diff --git a/api/src/org/labkey/api/data/CompareType.java b/api/src/org/labkey/api/data/CompareType.java index d9165ae67da..954a35739fa 100644 --- a/api/src/org/labkey/api/data/CompareType.java +++ b/api/src/org/labkey/api/data/CompareType.java @@ -827,6 +827,39 @@ protected Collection getCollectionParam(Object value) * * */ + + + public static final CompareType ARRAY_IS_EMPTY = new CompareType("Is Empty", "arrayisempty", "ARRAYISEMPTY", false, null, OperatorType.ARRAYISEMPTY) + { + @Override + public ArrayIsEmptyClause createFilterClause(@NotNull FieldKey fieldKey, Object value) + { + return new ArrayIsEmptyClause(fieldKey); + } + + @Override + public boolean meetsCriteria(ColumnRenderProperties col, Object value, Object[] filterValues) + { + throw new UnsupportedOperationException("Conditional formatting not yet supported for Multi Choices"); + } + }; + + + public static final CompareType ARRAY_IS_NOT_EMPTY = new CompareType("Is Not Empty", "arrayisnotempty", "ARRAYISNOTEMPTY", false, null, OperatorType.ARRAYISNOTEMPTY) + { + @Override + public ArrayIsEmptyClause createFilterClause(@NotNull FieldKey fieldKey, Object value) + { + return new ArrayIsNotEmptyClause(fieldKey); + } + + @Override + public boolean meetsCriteria(ColumnRenderProperties col, Object value, Object[] filterValues) + { + throw new UnsupportedOperationException("Conditional formatting not yet supported for Multi Choices"); + } + }; + public static final CompareType ARRAY_CONTAINS_ALL = new CompareType("Contains All", "arraycontainsall", "ARRAYCONTAINSALL", true, null, OperatorType.ARRAYCONTAINSALL) { @Override @@ -981,6 +1014,68 @@ public Pair getSqlFragments(Map columnMap, SqlDialect dialect) + { + ColumnInfo colInfo = columnMap != null ? columnMap.get(_fieldKey) : null; + var alias = SimpleFilter.getAliasForColumnFilter(dialect, colInfo, _fieldKey); + + SQLFragment columnFragment = new SQLFragment().appendIdentifier(alias); + + SQLFragment sql = dialect.array_is_empty(columnFragment); + if (!_negated) + return sql; + return new SQLFragment(" NOT (").append(sql).append(")"); + } + + @Override + public String getLabKeySQLWhereClause(Map columnMap) + { + return "array_is_empty(" + getLabKeySQLColName(_fieldKey) + ")"; + } + + @Override + public void appendFilterText(StringBuilder sb, ColumnNameFormatter formatter) + { + sb.append("is empty"); + } + + } + + private static class ArrayIsNotEmptyClause extends ArrayIsEmptyClause + { + + public ArrayIsNotEmptyClause(@NotNull FieldKey fieldKey) + { + super(fieldKey, CompareType.ARRAY_IS_NOT_EMPTY, true); + } + + @Override + public String getLabKeySQLWhereClause(Map columnMap) + { + return "NOT array_is_empty(" + getLabKeySQLColName(_fieldKey) + ")"; + } + + @Override + public void appendFilterText(StringBuilder sb, ColumnNameFormatter formatter) + { + sb.append("is not empty"); + } + + } + private static class ArrayContainsAllClause extends ArrayClause { diff --git a/api/src/org/labkey/api/data/SimpleFilter.java b/api/src/org/labkey/api/data/SimpleFilter.java index cdedb78ce2c..1cb5380207e 100644 --- a/api/src/org/labkey/api/data/SimpleFilter.java +++ b/api/src/org/labkey/api/data/SimpleFilter.java @@ -620,7 +620,7 @@ public static abstract class MultiValuedFilterClause extends CompareType.Abstrac public MultiValuedFilterClause(@NotNull FieldKey fieldKey, CompareType comparison, Collection params, boolean negated) { super(fieldKey); - params = new ArrayList<>(params); // possibly immutable + params = params == null ? new ArrayList<>() : new ArrayList<>(params); // possibly immutable if (params.contains(null)) //params.size() == 0 || { _includeNull = true; diff --git a/api/src/org/labkey/api/data/TableChange.java b/api/src/org/labkey/api/data/TableChange.java index 5e8c6a1245d..1ae2dd6d094 100644 --- a/api/src/org/labkey/api/data/TableChange.java +++ b/api/src/org/labkey/api/data/TableChange.java @@ -20,6 +20,7 @@ import org.labkey.api.data.PropertyStorageSpec.Index; import org.labkey.api.data.TableInfo.IndexDefinition; import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainKind; import org.labkey.api.util.logging.LogHelper; @@ -58,6 +59,7 @@ public class TableChange private Collection _constraints; private Set _indicesToBeDroppedByName; private IndexSizeMode _sizeMode = IndexSizeMode.Auto; + private Map _oldPropTypes; /** In most cases, domain knows the storage table name **/ public TableChange(Domain domain, ChangeType changeType) @@ -329,6 +331,16 @@ public void setForeignKeys(Collection foreignKey _foreignKeys = foreignKeys; } + public Map getOldPropTypes() + { + return _oldPropTypes; + } + + public void setOldPropTypes(Map oldPropTypes) + { + _oldPropTypes = oldPropTypes; + } + public final List toSpecs(Collection columnNames) { final Domain domain = _domain; @@ -349,6 +361,11 @@ public final List toSpecs(Collection columnNames) .collect(Collectors.toList()); } + public void setOldPropertyTypes(Map oldPropTypes) + { + _oldPropTypes = oldPropTypes; + } + public enum ChangeType { CreateTable, diff --git a/api/src/org/labkey/api/data/dialect/SqlDialect.java b/api/src/org/labkey/api/data/dialect/SqlDialect.java index 85cc60ec933..77e7430d148 100644 --- a/api/src/org/labkey/api/data/dialect/SqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/SqlDialect.java @@ -2200,6 +2200,12 @@ public SQLFragment array_construct(SQLFragment[] elements) throw new UnsupportedOperationException(getClass().getSimpleName() + " does not implement"); } + public SQLFragment array_is_empty(SQLFragment a) + { + assert !supportsArrays(); + throw new UnsupportedOperationException(getClass().getSimpleName() + " does not implement"); + } + // element a is in array b public SQLFragment element_in_array(SQLFragment a, SQLFragment b) { diff --git a/api/src/org/labkey/api/query/AbstractQueryChangeListener.java b/api/src/org/labkey/api/query/AbstractQueryChangeListener.java index 57c76f78dd3..4e4eb2ac135 100644 --- a/api/src/org/labkey/api/query/AbstractQueryChangeListener.java +++ b/api/src/org/labkey/api/query/AbstractQueryChangeListener.java @@ -40,13 +40,13 @@ public void queryCreated(User user, Container container, ContainerFilter scope, protected abstract void queryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, String query); @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { for (QueryPropertyChange change : changes) - queryChanged(user, container, scope, schema, change); + queryChanged(user, container, scope, schema, queryName, change); } - protected abstract void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, QueryPropertyChange change); + protected abstract void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, QueryPropertyChange change); @Override public void queryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) diff --git a/api/src/org/labkey/api/query/QueryChangeListener.java b/api/src/org/labkey/api/query/QueryChangeListener.java index 59d96dc79ee..06a80b681b9 100644 --- a/api/src/org/labkey/api/query/QueryChangeListener.java +++ b/api/src/org/labkey/api/query/QueryChangeListener.java @@ -20,10 +20,14 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; import org.labkey.api.event.PropertyChange; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; import org.labkey.api.security.User; import java.util.Collection; import java.util.Collections; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Listener for table and query events that fires when the structure/schema changes, but not when individual data @@ -58,10 +62,11 @@ public interface QueryChangeListener * @param container The container the tables or queries are changed in. * @param scope The scope of containers that the tables or queries affect. * @param schema The schema of the tables or queries. + * @param queryName The query name if the change is specific to a single query. * @param property The QueryProperty that has changed. * @param changes The set of change events. Each QueryPropertyChange is associated with a single table or query. */ - void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes); + void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @Nullable String queryName, @NotNull QueryProperty property, @NotNull Collection> changes); /** * This method is called when a set of tables or queries are deleted from the given container and schema. @@ -94,7 +99,9 @@ enum QueryProperty Description(String.class), Inherit(Boolean.class), Hidden(Boolean.class), - SchemaName(String.class),; + SchemaName(String.class), + ColumnName(String.class), + ColumnType(PropertyType.class),; private final Class _klass; @@ -112,7 +119,7 @@ public Class getPropertyClass() /** * A change event for a single property of a single table or query. * If multiple properties have been changed, QueryChangeListener will - * fire {@link QueryChangeListener#queryChanged(User, Container, ContainerFilter, SchemaKey, QueryChangeListener.QueryProperty, Collection)} + * fire {@link QueryChangeListener#queryChanged(User, Container, ContainerFilter, SchemaKey, String, QueryChangeListener.QueryProperty, Collection)} * for each property that has changed. * * @param The property type. @@ -171,6 +178,22 @@ public static void handleSchemaNameChange(@NotNull String oldValue, String newVa QueryProperty.SchemaName, Collections.singleton(change)); } + public static void handleColumnTypeChange(@NotNull PropertyDescriptor oldValue, PropertyDescriptor newValue, @NotNull SchemaKey schemaPath, @NotNull String queryName, User user, Container container) + { + if (oldValue.getPropertyType() == newValue.getPropertyType()) + return; + + QueryChangeListener.QueryPropertyChange change = new QueryChangeListener.QueryPropertyChange<>( + null, + QueryChangeListener.QueryProperty.ColumnType, + oldValue, + newValue + ); + + QueryService.get().fireQueryColumnChanged(user, container, schemaPath, queryName, + QueryProperty.ColumnType, Collections.singleton(change)); + } + @Nullable public QueryDefinition getSource() { return _queryDef; } @Override @@ -185,4 +208,156 @@ public static void handleSchemaNameChange(@NotNull String oldValue, String newVa @Nullable public V getNewValue() { return _newValue; } } + + /** + * Utility to update encoded filter string when a column type changes from Multi_Choice to a non Multi_Choice. + * This method performs targeted replacements for the given column name (case-insensitive). + */ + private static String getUpdatedFilterStrFromMVTC(String filterStr, String columnName, PropertyType oldType, PropertyType newType) + { + if (filterStr == null || columnName == null || oldType == null || newType == null) + return filterStr; + + // Only act when changing away from MULTI_CHOICE + if (oldType != PropertyType.MULTI_CHOICE || newType == PropertyType.MULTI_CHOICE) + return filterStr; + + String colLower = columnName.toLowerCase(); + String sLower = filterStr.toLowerCase(); + + // No action if column doesn't match + if (!sLower.startsWith("filter." + colLower + "~")) + return filterStr; + + // drop arraycontainsall since there is no good match + if (sLower.startsWith("filter." + colLower + "~arraycontainsall")) + return ""; + + String updated = filterStr; + + if (containsOp(updated, columnName, "arrayisempty")) + { + return replaceOp(updated, columnName, "arrayisempty", "isblank"); + } + if (containsOp(updated, columnName, "arrayisnotempty")) + { + return replaceOp(updated, columnName, "arrayisnotempty", "isnonblank"); + } + if (containsOp(updated, columnName, "arraymatches")) + { + updated = replaceOp(updated, columnName, "arraymatches", "eq"); + // Replace all occurrences of %2C with %2C%20, + // "," -> ", " during array to string conversion + return updated.replace("%2C", "%2C%20"); + } + if (containsOp(updated, columnName, "arraynotmatches")) + { + updated = replaceOp(updated, columnName, "arraynotmatches", "neq"); + // Replace all occurrences of %2C with %2C%20 + return updated.replace("%2C", "%2C%20"); + } + if (containsOp(updated, columnName, "arraycontainsany")) + { + updated = replaceOp(updated, columnName, "arraycontainsany", "in"); + // Replace all occurrences of %2C with %3B + // ";" is used as the separator for "in" operator + return updated.replace("%2C", "%3B"); + } + if (containsOp(updated, columnName, "arraycontainsnone")) + { + updated = replaceOp(updated, columnName, "arraycontainsnone", "notin"); + // Replace all occurrences of %2C with %3B + // ";" is used as the separator for "notin" operator + return updated.replace("%2C", "%3B"); + } + + // No matching operator found for this column, drop the filter + return ""; + } + + /** + * Utility to update encoded filter string when a column type is changed to Multi_Choice (migrating operators to array equivalents). + */ + private static String getUpdatedMVTCFilterStr(String filterStr, String columnName, PropertyType oldType, PropertyType newType) + { + if (filterStr == null || columnName == null || oldType == null || newType == null) + return filterStr; + + // Only act when changing to MULTI_CHOICE + if (oldType == PropertyType.MULTI_CHOICE || newType != PropertyType.MULTI_CHOICE) + return filterStr; + + String colLower = columnName.toLowerCase(); + String sLower = filterStr.toLowerCase(); + + // No action if column doesn't match + if (!sLower.startsWith("filter." + colLower + "~")) + return filterStr; + + String updated = filterStr; + + // Return on first matching operator for this column + if (containsOp(updated, columnName, "eq")) + { + return replaceOp(updated, columnName, "eq", "arraymatches"); + } + if (containsOp(updated, columnName, "neq")) + { + return replaceOp(updated, columnName, "neq", "arraycontainsnone"); + } + if (containsOp(updated, columnName, "isblank")) + { + return replaceOp(updated, columnName, "isblank", "arrayisempty"); + } + if (containsOp(updated, columnName, "isnonblank")) + { + return replaceOp(updated, columnName, "isnonblank", "arrayisnotempty"); + } + if (containsOp(updated, columnName, "in")) + { + updated = replaceOp(updated, columnName, "in", "arraycontainsany"); + // update ";" to "," for separator appropriate for array operator + return updated.replace("%3B", "%2C"); + } + if (containsOp(updated, columnName, "notin")) + { + updated = replaceOp(updated, columnName, "notin", "arraycontainsnone"); + return updated.replace("%3B", "%2C"); + } + + // No matching operator found for this column, drop the filter + return ""; + } + + static String getUpdatedFilterStrOnColumnTypeUpdate(String filterStr, String columnName, PropertyType oldType, PropertyType newType) + { + if (oldType == PropertyType.MULTI_CHOICE) + return getUpdatedFilterStrFromMVTC(filterStr, columnName, oldType, newType); + else if (newType == PropertyType.MULTI_CHOICE) + return getUpdatedMVTCFilterStr(filterStr, columnName, oldType, newType); + else + return filterStr; + } + + private static boolean containsOp(String filterStr, String columnName, String op) + { + String regex = "(?i)filter\\." + Pattern.quote(columnName) + "~" + Pattern.quote(op); + return Pattern.compile(regex).matcher(filterStr).find(); + } + + private static String replaceOp(String filterStr, String columnName, String fromOp, String toOp) + { + String regex = "(?i)(filter\\.)" + Pattern.quote(columnName) + "(~)" + Pattern.quote(fromOp); + Matcher m = Pattern.compile(regex).matcher(filterStr); + StringBuffer sb = new StringBuffer(); + while (m.find()) + { + // Preserve the literal 'filter.' and '~', but use the provided columnName casing and new operator + String replacement = m.group(1) + columnName + m.group(2) + toOp; + m.appendReplacement(sb, Matcher.quoteReplacement(replacement)); + } + m.appendTail(sb); + return sb.toString(); + } + } diff --git a/api/src/org/labkey/api/query/QueryService.java b/api/src/org/labkey/api/query/QueryService.java index bb6d87b7614..942d6a79461 100644 --- a/api/src/org/labkey/api/query/QueryService.java +++ b/api/src/org/labkey/api/query/QueryService.java @@ -491,7 +491,7 @@ public String getDefaultCommentSummary() void fireQueryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries); void fireQueryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, QueryChangeListener.QueryProperty property, Collection> changes); void fireQueryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries); - + void fireQueryColumnChanged(User user, Container container, @NotNull SchemaKey schemaPath, @NotNull String queryName, QueryChangeListener.QueryProperty property, Collection> changes); /** OLAP **/ // could make this a separate service diff --git a/core/package-lock.json b/core/package-lock.json index c9c708e80a2..7897498f8de 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.13.0", + "@labkey/components": "7.14.0-fb-mvtc-convert.3", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.13.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.13.0.tgz", - "integrity": "sha512-+2o42no7q9IInKbvSd5XHDrnmLKucgudQ+7C2FD6ya+Da8mRu76GWG6L168iwbtMaguQZzFQmMGpD5VScWZiyQ==", + "version": "7.14.0-fb-mvtc-convert.3", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.14.0-fb-mvtc-convert.3.tgz", + "integrity": "sha512-zWFCmFERVct/gyKuOMZTylR7CEBgpcsOnU8A3sjfX/gXyxJ/JgC5BsFSOe92zOyMYRmMkNoBa+m+9QwTqAom6Q==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index 2c1202f3221..d9b435baf3e 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.13.0", + "@labkey/components": "7.14.0-fb-mvtc-convert.3", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java index c5fbcad586b..daca9198496 100644 --- a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java +++ b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java @@ -43,6 +43,7 @@ import org.labkey.api.data.dialect.JdbcHelper; import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.data.dialect.StandardJdbcHelper; +import org.labkey.api.exp.PropertyType; import org.labkey.api.query.AliasManager; import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.HtmlString; @@ -630,6 +631,7 @@ private List getChangeColumnTypeStatement(TableChange change) for (PropertyStorageSpec column : change.getColumns()) { + PropertyType oldPropertyType = change.getOldPropTypes().get(column.getName()); DatabaseIdentifier columnIdent = makePropertyIdentifier(column.getName()); if (column.getJdbcType().isDateOrTime()) { @@ -661,6 +663,76 @@ private List getChangeColumnTypeStatement(TableChange change) rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); statements.add(rename); } + else if (oldPropertyType == PropertyType.MULTI_CHOICE && column.getJdbcType().isText()) + { + // Converting from text[] (array) to text requires an intermediate column and transformation + String tempColumnName = column.getName() + "~~temp~~"; + DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); + + // 1) ADD temp column of text type + SQLFragment addTemp = new SQLFragment("ALTER TABLE "); + addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); + statements.add(addTemp); + + // 2) UPDATE: convert and copy value to temp column + // - NULL array -> NULL + // - empty array -> NULL + // - non-empty array -> concatenate array elements with comma (', ') + SQLFragment update = new SQLFragment("UPDATE "); + update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + update.append(" SET ").appendIdentifier(tempColumnIdent).append(" = CASE "); + update.append(" WHEN ").appendIdentifier(columnIdent).append(" IS NULL THEN NULL "); + update.append(" WHEN COALESCE(array_length(").appendIdentifier(columnIdent).append(", 1), 0) = 0 THEN NULL "); + update.append(" ELSE array_to_string(").appendIdentifier(columnIdent).append(", ', ') END"); + statements.add(update); + + // 3) DROP original column + SQLFragment drop = new SQLFragment("ALTER TABLE "); + drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); + statements.add(drop); + + // 4) RENAME temp column to original column name + SQLFragment rename = new SQLFragment("ALTER TABLE "); + rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); + statements.add(rename); + } + else if (column.getJdbcType() == JdbcType.ARRAY) + { + // Converting from text to text[] requires an intermediate column and transformation + String tempColumnName = column.getName() + "~~temp~~"; + DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); + + // 1) ADD temp column of array type (e.g., text[]) + SQLFragment addTemp = new SQLFragment("ALTER TABLE "); + addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); + statements.add(addTemp); + + // 2) UPDATE: copy converted value to temp column as single-element array + // - NULL or blank ('') -> empty array [] + // - otherwise -> single-element array [text] + SQLFragment update = new SQLFragment("UPDATE "); + update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + update.append(" SET ").appendIdentifier(tempColumnIdent); + update.append(" = CASE WHEN ").appendIdentifier(columnIdent).append(" IS NULL OR ").appendIdentifier(columnIdent).append(" = '' THEN ARRAY[]::text[] ELSE ARRAY["); + update.appendIdentifier(columnIdent).append("]::text[] END"); + statements.add(update); + + // 3) DROP original column + SQLFragment drop = new SQLFragment("ALTER TABLE "); + drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); + statements.add(drop); + + // 4) RENAME temp column to original column name + SQLFragment rename = new SQLFragment("ALTER TABLE "); + rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); + statements.add(rename); + } else { String dbType; @@ -1085,6 +1157,12 @@ public SQLFragment array_construct(SQLFragment[] elements) return ret; } + @Override + public SQLFragment array_is_empty(SQLFragment a) + { + return new SQLFragment("(cardinality(").append(a).append(")=0)"); + } + @Override public SQLFragment array_all_in_array(SQLFragment a, SQLFragment b) { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index c3ed56aedca..94a68fea867 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.13.0" + "@labkey/components": "7.14.0-fb-mvtc-convert.3" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.13.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.13.0.tgz", - "integrity": "sha512-+2o42no7q9IInKbvSd5XHDrnmLKucgudQ+7C2FD6ya+Da8mRu76GWG6L168iwbtMaguQZzFQmMGpD5VScWZiyQ==", + "version": "7.14.0-fb-mvtc-convert.3", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.14.0-fb-mvtc-convert.3.tgz", + "integrity": "sha512-zWFCmFERVct/gyKuOMZTylR7CEBgpcsOnU8A3sjfX/gXyxJ/JgC5BsFSOe92zOyMYRmMkNoBa+m+9QwTqAom6Q==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index 6601245661c..d98c3e473ac 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.13.0" + "@labkey/components": "7.14.0-fb-mvtc-convert.3" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/experiment/src/org/labkey/experiment/ExperimentQueryChangeListener.java b/experiment/src/org/labkey/experiment/ExperimentQueryChangeListener.java index f0a6c6d5201..177dc34ab45 100644 --- a/experiment/src/org/labkey/experiment/ExperimentQueryChangeListener.java +++ b/experiment/src/org/labkey/experiment/ExperimentQueryChangeListener.java @@ -63,7 +63,7 @@ private List getRenamedDataClasses(Container container, String } @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { boolean isSamples = schema.toString().equalsIgnoreCase("samples"); boolean isData = schema.toString().equalsIgnoreCase("exp.data"); diff --git a/experiment/src/org/labkey/experiment/PropertyQueryChangeListener.java b/experiment/src/org/labkey/experiment/PropertyQueryChangeListener.java index 65e879b53bd..e29ccc60340 100644 --- a/experiment/src/org/labkey/experiment/PropertyQueryChangeListener.java +++ b/experiment/src/org/labkey/experiment/PropertyQueryChangeListener.java @@ -61,7 +61,7 @@ private void updateLookupSchema(String newValue, String oldSchema, Container con } @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { if (!property.equals(QueryProperty.SchemaName) && !property.equals(QueryProperty.Name)) // Issue 53846 return; diff --git a/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java b/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java index 2b10c8c8718..1715b088f72 100644 --- a/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java +++ b/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java @@ -26,6 +26,7 @@ import org.labkey.api.data.ColumnRenderPropertiesImpl; import org.labkey.api.data.ConditionalFormat; import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.DatabaseIdentifier; import org.labkey.api.data.JdbcType; @@ -33,6 +34,7 @@ import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.exp.ChangePropertyDescriptorException; import org.labkey.api.exp.DomainDescriptor; @@ -52,6 +54,8 @@ import org.labkey.api.gwt.client.DefaultScaleType; import org.labkey.api.gwt.client.DefaultValueType; import org.labkey.api.gwt.client.FacetingBehaviorType; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.SchemaKey; import org.labkey.api.security.User; import org.labkey.api.util.StringExpressionFactory; import org.labkey.api.util.TestContext; @@ -840,6 +844,11 @@ else if (newType.getJdbcType().isDateOrTime() && oldType.getJdbcType().isDateOrT changedType = true; _pd.setFormat(null); } + else if (newType == PropertyType.MULTI_CHOICE || oldType == PropertyType.MULTI_CHOICE) + { + changedType = true; + _pd.setFormat(null); + } else { throw new ChangePropertyDescriptorException("Cannot convert an instance of " + oldType.getJdbcType() + " to " + newType.getJdbcType() + "."); @@ -873,13 +882,21 @@ else if (newType.getJdbcType().isDateOrTime() && oldType.getJdbcType().isDateOrT if (changedType) { + var domainKind = _domain.getDomainKind(); + if (domainKind == null) + throw new ChangePropertyDescriptorException("Cannot change property type for domain, unknown domain kind."); + StorageProvisionerImpl.get().changePropertyType(this.getDomain(), this); if (_pdOld.getJdbcType() == JdbcType.BOOLEAN && _pd.getJdbcType().isText()) { updateBooleanValue( - new SQLFragment().appendIdentifier(_domain.getDomainKind().getStorageSchemaName()).append(".").appendIdentifier(_domain.getStorageTableName()), + new SQLFragment().appendIdentifier(domainKind.getStorageSchemaName()).append(".").appendIdentifier(_domain.getStorageTableName()), _pd.getLegalSelectName(dialect), _pdOld.getFormat(), null); // GitHub Issue #647 } + + TableInfo table = domainKind.getTableInfo(user, getContainer(), _domain, ContainerFilter.getUnsafeEverythingFilter()); + if (table != null && _pdOld.getPropertyType() != null) + QueryChangeListener.QueryPropertyChange.handleColumnTypeChange(_pdOld, _pd, SchemaKey.fromString(table.getUserSchema().getName()), table.getName(), user, getContainer()); } else if (propResized) StorageProvisionerImpl.get().resizeProperty(this.getDomain(), this, _pdOld.getScale()); diff --git a/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java b/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java index 727423a1a70..8d588a3e4c1 100644 --- a/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java +++ b/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java @@ -66,6 +66,7 @@ import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.PropertyColumn; import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; import org.labkey.api.exp.api.ExperimentUrls; import org.labkey.api.exp.api.StorageProvisioner; import org.labkey.api.exp.property.AbstractDomainKind; @@ -112,6 +113,8 @@ import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; + /** * Creates and maintains "hard" tables in the underlying database based on dynamically configured data types. * Will do CREATE TABLE and ALTER TABLE statements to make sure the table has the right set of requested columns. @@ -573,9 +576,35 @@ public void changePropertyType(Domain domain, DomainProperty prop) throws Change Set base = Sets.newCaseInsensitiveHashSet(); kind.getBaseProperties(domain).forEach(s -> base.add(s.getName())); + Map oldPropTypes = new HashMap<>(); if (!base.contains(prop.getName())) + { + if (prop instanceof DomainPropertyImpl dpi) + { + var oldPd = dpi._pdOld; + if (oldPd != null) + { + var newPd = dpi._pd; + if (oldPd.getPropertyType() == PropertyType.MULTI_CHOICE && TEXT_CHOICE_CONCEPT_URI.equals(newPd.getConceptURI())) + { + String sql = "SELECT COUNT(*) FROM " + kind.getStorageSchemaName() + "." + domain.getStorageTableName() + + " WHERE " + prop.getPropertyDescriptor().getStorageColumnName() + " IS NOT NULL AND " + + " array_length(" + prop.getPropertyDescriptor().getStorageColumnName() + ", 1) > 1"; + long count = new SqlSelector(scope, sql).getObject(Long.class); + if (count > 0) + { + throw new ChangePropertyDescriptorException("Unable to change property type. There are rows with multiple values stored for '" + prop.getName() + "'."); + } + } + oldPropTypes.put(prop.getName(), oldPd.getPropertyType()); + } + + } + propChange.addColumn(prop.getPropertyDescriptor()); + } + propChange.setOldPropertyTypes(oldPropTypes); propChange.execute(); } diff --git a/query/src/org/labkey/query/CustomViewQueryChangeListener.java b/query/src/org/labkey/query/CustomViewQueryChangeListener.java index d1c16e0fee3..135dad9dab1 100644 --- a/query/src/org/labkey/query/CustomViewQueryChangeListener.java +++ b/query/src/org/labkey/query/CustomViewQueryChangeListener.java @@ -20,6 +20,8 @@ import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.query.CustomView; import org.labkey.api.query.CustomViewChangeListener; import org.labkey.api.query.CustomViewInfo; @@ -28,6 +30,10 @@ import org.labkey.api.query.QueryService; import org.labkey.api.query.SchemaKey; import org.labkey.api.security.User; +import org.labkey.api.exp.PropertyType; + +import java.util.regex.Pattern; +import java.util.regex.Matcher; import org.springframework.mock.web.MockHttpServletRequest; import jakarta.servlet.http.HttpServletRequest; @@ -55,7 +61,7 @@ public void queryCreated(User user, Container container, ContainerFilter scope, } @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { if (property.equals(QueryProperty.Name)) { @@ -65,6 +71,64 @@ public void queryChanged(User user, Container container, ContainerFilter scope, { _updateCustomViewSchemaNameChange(user, container, changes); } + if (property.equals(QueryProperty.ColumnType)) + { + _updateCustomViewColumnTypeChange(user, container, schema, queryName, changes); + } + } + + + private void _updateCustomViewColumnTypeChange(User user, Container container, SchemaKey schema, String queryName, @NotNull Collection> changes) + { + for (QueryPropertyChange qpc : changes) + { + + PropertyDescriptor oldDp = (PropertyDescriptor) qpc.getOldValue(); + PropertyDescriptor newDp = (PropertyDescriptor) qpc.getNewValue(); + + if (oldDp == null || newDp == null) + continue; + + String columnName = newDp.getName() == null ? oldDp.getName() : newDp.getName(); + + List databaseCustomViews = QueryService.get().getDatabaseCustomViews(user, container, null, schema.toString(), queryName, false, false); + + for (CustomView customView : databaseCustomViews) + { + try + { + // update custom view filter and sort based on column type change + String filterAndSort = customView.getFilterAndSort(); + if (filterAndSort == null || filterAndSort.isEmpty()) + continue; + + /* Example: + * "/?filter.MCF2~arrayisnotempty=&filter.Name~in=S-5%3BS-6%3BS-8%3BS-9&filter.MCF~arraycontainsall=2%2C1%2C3&filter.sort=zz" + */ + String prefix = filterAndSort.startsWith("/?") ? "/?" : (filterAndSort.startsWith("?") ? "?" : ""); + String[] filterComponents = filterAndSort.substring(prefix.length()).split("&"); + StringBuilder updatedFilterAndSort = new StringBuilder(prefix); + for (String filterPart : filterComponents) + { + String updatedPart = QueryChangeListener.getUpdatedFilterStrOnColumnTypeUpdate(filterPart, columnName, oldDp.getPropertyType(), newDp.getPropertyType()); + updatedFilterAndSort.append(updatedPart); + } + + String updatedFilterAndSortStr = updatedFilterAndSort.toString(); + if (!updatedFilterAndSortStr.equals(filterAndSort)) + { + customView.setFilterAndSort(updatedFilterAndSortStr); + HttpServletRequest request = new MockHttpServletRequest(); + customView.save(customView.getModifiedBy(), request); + } + } + catch (Exception e) + { + LogManager.getLogger(CustomViewQueryChangeListener.class).error("An error occurred upgrading custom view properties: ", e); + } + } + } + } @Override diff --git a/query/src/org/labkey/query/QueryDefQueryChangeListener.java b/query/src/org/labkey/query/QueryDefQueryChangeListener.java index ec74cb74dc3..8f9ae84eae9 100644 --- a/query/src/org/labkey/query/QueryDefQueryChangeListener.java +++ b/query/src/org/labkey/query/QueryDefQueryChangeListener.java @@ -20,7 +20,7 @@ public void queryCreated(User user, Container container, ContainerFilter scope, {} @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { if (property.equals(QueryProperty.Name)) { diff --git a/query/src/org/labkey/query/QueryServiceImpl.java b/query/src/org/labkey/query/QueryServiceImpl.java index d44943d4f2a..e361786fa82 100644 --- a/query/src/org/labkey/query/QueryServiceImpl.java +++ b/query/src/org/labkey/query/QueryServiceImpl.java @@ -309,6 +309,8 @@ public void moduleChanged(Module module) CompareType.NONBLANK, CompareType.MV_INDICATOR, CompareType.NO_MV_INDICATOR, + CompareType.ARRAY_IS_EMPTY, + CompareType.ARRAY_IS_NOT_EMPTY, CompareType.ARRAY_CONTAINS_ALL, CompareType.ARRAY_CONTAINS_ANY, CompareType.ARRAY_CONTAINS_NONE, @@ -3268,7 +3270,13 @@ public void fireQueryCreated(User user, Container container, ContainerFilter sco @Override public void fireQueryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, QueryChangeListener.QueryProperty property, Collection> changes) { - QueryManager.get().fireQueryChanged(user, container, scope, schema, property, changes); + QueryManager.get().fireQueryChanged(user, container, scope, schema, null, property, changes); + } + + @Override + public void fireQueryColumnChanged(User user, Container container, @NotNull SchemaKey schemaPath, @NotNull String queryName, QueryChangeListener.QueryProperty property, Collection> changes) + { + QueryManager.get().fireQueryChanged(user, container, null, schemaPath, queryName, property, changes); } @Override diff --git a/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java b/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java index ef497ad7be1..4d20461cb46 100644 --- a/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java +++ b/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java @@ -44,7 +44,7 @@ public void queryCreated(User user, Container container, ContainerFilter scope, } @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { if (property.equals(QueryProperty.Name)) { diff --git a/query/src/org/labkey/query/persist/QueryManager.java b/query/src/org/labkey/query/persist/QueryManager.java index 28e10d84736..c041d8c9a2d 100644 --- a/query/src/org/labkey/query/persist/QueryManager.java +++ b/query/src/org/labkey/query/persist/QueryManager.java @@ -518,12 +518,12 @@ public void fireQueryCreated(User user, Container container, ContainerFilter sco l.queryCreated(user, container, scope, schema, queries); } - public void fireQueryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryChangeListener.QueryProperty property, @NotNull Collection> changes) + public void fireQueryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @Nullable String queryName, @NotNull QueryChangeListener.QueryProperty property, @NotNull Collection> changes) { QueryService.get().updateLastModified(); assert checkChanges(property, changes); for (QueryChangeListener l : QUERY_LISTENERS) - l.queryChanged(user, container, scope, schema, property, changes); + l.queryChanged(user, container, scope, schema, queryName, property, changes); } // Checks all changes have the correct property and type. diff --git a/query/src/org/labkey/query/reports/ReportQueryChangeListener.java b/query/src/org/labkey/query/reports/ReportQueryChangeListener.java index 57ff5483034..f4c05a89536 100644 --- a/query/src/org/labkey/query/reports/ReportQueryChangeListener.java +++ b/query/src/org/labkey/query/reports/ReportQueryChangeListener.java @@ -75,7 +75,7 @@ public void queryCreated(User user, Container container, ContainerFilter scope, } @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { if (property.equals(QueryProperty.Name)) { diff --git a/query/src/org/labkey/query/sql/Method.java b/query/src/org/labkey/query/sql/Method.java index 77214c9d348..b6377660675 100644 --- a/query/src/org/labkey/query/sql/Method.java +++ b/query/src/org/labkey/query/sql/Method.java @@ -1573,6 +1573,34 @@ public SQLFragment getSQL(SqlDialect dialect, SQLFragment[] arguments) } } + public static class ArrayIsEmptyMethod extends Method + { + ArrayIsEmptyMethod(String name) + { + super(name, JdbcType.BOOLEAN, 1, 1); + } + + @Override + public MethodInfo getMethodInfo() + { + return new AbstractMethodInfo(JdbcType.BOOLEAN) + { + @Override + public JdbcType getJdbcType(JdbcType[] args) + { + if (1 == args.length && args[0] != JdbcType.ARRAY) + throw new QueryParseException(_name + " requires an argument of type ARRAY", null, -1, -1); + return super.getJdbcType(args); + } + + @Override + public SQLFragment getSQL(SqlDialect dialect, SQLFragment[] arguments) + { + return dialect.array_is_empty(arguments[0]); + } + }; + } + } final static Map postgresMethods = Collections.synchronizedMap(new CaseInsensitiveHashMap<>()); @@ -1650,6 +1678,8 @@ private static void addPostgresArrayMethods() // not array_equals() because arrays are ordered, this is an unordered comparison postgresMethods.put("array_is_same", new ArrayOperatorMethod("array_is_same", SqlDialect::array_same_array)); // Use "NOT array_is_same()" instead of something clumsy like "array_is_not_same()" + + postgresMethods.put("array_is_empty", new ArrayIsEmptyMethod("array_is_empty")); } diff --git a/study/src/org/labkey/study/query/QueryDatasetQueryChangeListener.java b/study/src/org/labkey/study/query/QueryDatasetQueryChangeListener.java index 043bf89c69d..fa4823fd99b 100644 --- a/study/src/org/labkey/study/query/QueryDatasetQueryChangeListener.java +++ b/study/src/org/labkey/study/query/QueryDatasetQueryChangeListener.java @@ -25,7 +25,7 @@ public void queryCreated(User user, Container container, ContainerFilter scope, } @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { if (property.equals(QueryProperty.Name)) {