Skip to content

Commit a7cae04

Browse files
labkey-teamcitylabkey-teamcity
committed
Merge 26.4 to develop
2 parents 3f658dd + f3ba583 commit a7cae04

File tree

4 files changed

+256
-6
lines changed

4 files changed

+256
-6
lines changed

‎luminex/src/org/labkey/luminex/LuminexModule.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,12 @@
2323
import org.labkey.api.assay.AssayQCFlagColumn;
2424
import org.labkey.api.assay.AssayService;
2525
import org.labkey.api.data.Container;
26+
import org.labkey.api.data.UpgradeCode;
2627
import org.labkey.api.exp.api.ExperimentService;
2728
import org.labkey.api.exp.property.PropertyService;
2829
import org.labkey.api.module.DefaultModule;
2930
import org.labkey.api.module.ModuleContext;
31+
import org.labkey.api.module.ModuleProperty;
3032
import org.labkey.api.view.WebPartFactory;
3133
import org.labkey.luminex.query.LuminexProtocolSchema;
3234

@@ -83,6 +85,12 @@ public void doStartup(ModuleContext moduleContext)
8385
PropertyService.get().registerDomainKind(new LuminexDataDomainKind());
8486

8587
AssayFlagHandler.registerHandler(AssayService.get().getProvider(LuminexAssayProvider.NAME), new AssayDefaultFlagHandler());
88+
89+
ModuleProperty leveyJenningsMinDateProp = new ModuleProperty(this, "leveyJenningsMinDate");
90+
leveyJenningsMinDateProp.setLabel("Levey-Jennings Report Min Date Filter");
91+
leveyJenningsMinDateProp.setDescription("If provided, the minimum acquisition date to use as a filter for the Luminex Levey-Jennings report data");
92+
leveyJenningsMinDateProp.setInputType(ModuleProperty.InputType.text);
93+
addModuleProperty(leveyJenningsMinDateProp);
8694
}
8795

8896
@Override
@@ -103,4 +111,10 @@ public Set<String> getSchemaNames()
103111
LuminexSaveExclusionsForm.TestCase.class
104112
);
105113
}
114+
115+
@Override
116+
public @Nullable UpgradeCode getUpgradeCode()
117+
{
118+
return new LuminexUpgradeCode();
119+
}
106120
}

‎luminex/src/org/labkey/luminex/LuminexUpgradeCode.java‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
package org.labkey.luminex;
2+
3+
import org.apache.commons.lang3.ArrayUtils;
4+
import org.apache.logging.log4j.Logger;
5+
import org.labkey.api.assay.AssayService;
6+
import org.labkey.api.collections.CaseInsensitiveHashMap;
7+
import org.labkey.api.data.BeanObjectFactory;
8+
import org.labkey.api.data.DbScope;
9+
import org.labkey.api.data.ObjectFactory;
10+
import org.labkey.api.data.SQLFragment;
11+
import org.labkey.api.data.SimpleFilter;
12+
import org.labkey.api.data.SqlSelector;
13+
import org.labkey.api.data.TableSelector;
14+
import org.labkey.api.data.UpgradeCode;
15+
import org.labkey.api.data.dialect.SqlDialect;
16+
import org.labkey.api.data.statistics.MathStat;
17+
import org.labkey.api.data.statistics.StatsService;
18+
import org.labkey.api.dataiterator.DataIteratorContext;
19+
import org.labkey.api.dataiterator.MapDataIterator;
20+
import org.labkey.api.exp.Lsid;
21+
import org.labkey.api.exp.OntologyManager;
22+
import org.labkey.api.exp.api.ExpProtocol;
23+
import org.labkey.api.exp.api.ExpRun;
24+
import org.labkey.api.exp.api.ExperimentService;
25+
import org.labkey.api.module.ModuleContext;
26+
import org.labkey.api.query.BatchValidationException;
27+
import org.labkey.api.query.FieldKey;
28+
import org.labkey.api.security.User;
29+
import org.labkey.api.util.GUID;
30+
import org.labkey.api.util.logging.LogHelper;
31+
import org.labkey.luminex.model.LuminexDataRow;
32+
import org.labkey.luminex.query.LuminexDataTable;
33+
import org.labkey.luminex.query.LuminexProtocolSchema;
34+
35+
import java.util.ArrayList;
36+
import java.util.HashMap;
37+
import java.util.List;
38+
import java.util.Map;
39+
40+
public class LuminexUpgradeCode implements UpgradeCode
41+
{
42+
private static final Logger LOG = LogHelper.getLogger(LuminexUpgradeCode.class, "Luminex upgrade code");
43+
44+
/**
45+
* GitHub Issue #875: Upgrade code to check for Luminex assay runs that have both summary and raw data but are missing summary rows.
46+
* NOTE: this upgrade code is not called from a SQL upgrade script. It is meant to be run manually from the admin console SQL Scripts page.
47+
*/
48+
public static void checkForMissingSummaryRows(ModuleContext ctx)
49+
{
50+
if (ctx.isNewInstall())
51+
return;
52+
53+
DbScope scope = LuminexProtocolSchema.getSchema().getScope();
54+
try (DbScope.Transaction tx = scope.ensureTransaction())
55+
{
56+
// For any Luminex dataids (input files) that have both summary and raw data rows,
57+
// find Luminex raw data rows (summary = false) that don't have a corresponding summary data row (summary = true)
58+
// NOTE: the d.created date filter is because the GitHub Issue 875 only applies to runs imported after this date
59+
SqlDialect dialect = LuminexProtocolSchema.getSchema().getSqlDialect();
60+
SQLFragment missingSummaryRowsSql = new SQLFragment("""
61+
SELECT DISTINCT d.runid, dr_false.dataid, dr_false.analyteid, dr_false.type
62+
FROM luminex.datarow dr_false
63+
LEFT JOIN exp.data d ON d.rowid = dr_false.dataid
64+
WHERE d.created > '2025-02-17'
65+
AND dr_false.summary = """).append(dialect.getBooleanFALSE()).append("\n").append("""
66+
AND EXISTS (SELECT 1 FROM luminex.datarow WHERE dataid = dr_false.dataid AND summary = """).append(dialect.getBooleanTRUE()).append(")\n").append("""
67+
AND EXISTS (SELECT 1 FROM luminex.datarow WHERE dataid = dr_false.dataid AND summary = """).append(dialect.getBooleanFALSE()).append(")\n").append("""
68+
AND NOT EXISTS (SELECT 1 FROM luminex.datarow dr_true
69+
WHERE dr_true.summary = """).append(dialect.getBooleanTRUE()).append("\n").append("""
70+
AND dr_true.dataid = dr_false.dataid
71+
AND dr_true.analyteid = dr_false.analyteid
72+
AND dr_true.type = dr_false.type
73+
)
74+
""");
75+
76+
int missingSummaryRowCount = new SqlSelector(scope, new SQLFragment("SELECT COUNT(*) FROM (").append(missingSummaryRowsSql).append(") as subq")).getObject(Integer.class);
77+
if (missingSummaryRowCount == 0)
78+
{
79+
LOG.info("No missing summary rows found for Luminex assay data.");
80+
return;
81+
}
82+
83+
new SqlSelector(scope, missingSummaryRowsSql).forEach(rs -> {
84+
int runid = rs.getInt("runid");
85+
int dataId = rs.getInt("dataid");
86+
int analyteId = rs.getInt("analyteid");
87+
String type = rs.getString("type");
88+
89+
ExpRun expRun = ExperimentService.get().getExpRun(runid);
90+
if (expRun == null)
91+
{
92+
LOG.warn("Could not find run for runid: " + runid + ", skipping missing summary row check for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type);
93+
return;
94+
}
95+
96+
LOG.info("Missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + " in run: " + expRun.getName() + " (" + expRun.getRowId() + ")");
97+
98+
// currently only inserting summary rows for Background (type = B) data rows
99+
if (!"B".equals(type))
100+
{
101+
LOG.warn("...not inserting missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + " because type is not 'B' (Background)");
102+
return;
103+
}
104+
105+
// Query for existing raw data rows with the same dataId, analyteId, and type
106+
StatsService service = StatsService.get();
107+
User user = ctx.getUpgradeUser();
108+
ExpProtocol protocol = expRun.getProtocol();
109+
LuminexDataTable tableInfo = ((LuminexProtocolSchema)AssayService.get().getProvider(protocol).createProtocolSchema(user, expRun.getContainer(), protocol, null)).createDataTable(null, false);
110+
SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Data"), dataId);
111+
filter.addCondition(FieldKey.fromParts("Analyte"), analyteId);
112+
filter.addCondition(FieldKey.fromParts("Type"), type);
113+
114+
// keep track of the set of wells for the given dataId/analyteId/type/standard combinations
115+
record WellGroupKey(long dataId, int analyteId, String type, String standard) {}
116+
Map<WellGroupKey, List<LuminexDataRow>> rowsByWellGroup = new HashMap<>();
117+
for (Map<String, Object> databaseMap : new TableSelector(tableInfo, filter, null).getMapCollection())
118+
{
119+
LuminexDataRow existingRow = BeanObjectFactory.Registry.getFactory(LuminexDataRow.class).fromMap(databaseMap);
120+
existingRow._setExtraProperties(new CaseInsensitiveHashMap<>(databaseMap));
121+
122+
WellGroupKey groupKey = new WellGroupKey(
123+
existingRow.getData(),
124+
existingRow.getAnalyte(),
125+
existingRow.getType(),
126+
(String) existingRow._getExtraProperties().get("Standard")
127+
);
128+
rowsByWellGroup.computeIfAbsent(groupKey, k -> new ArrayList<>()).add(existingRow);
129+
}
130+
131+
// calculate summary stats and well information for the new summary rows that we will insert into the database
132+
for (Map.Entry<WellGroupKey, List<LuminexDataRow>> wellGroupEntry : rowsByWellGroup.entrySet())
133+
{
134+
WellGroupKey groupKey = wellGroupEntry.getKey();
135+
LuminexDataRow newRow = new LuminexDataRow();
136+
newRow.setSummary(true);
137+
newRow.setData(groupKey.dataId);
138+
newRow.setAnalyte(groupKey.analyteId);
139+
newRow.setType(groupKey.type);
140+
141+
List<Double> fis = new ArrayList<>();
142+
List<Double> fiBkgds = new ArrayList<>();
143+
List<String> wells = new ArrayList<>();
144+
for (LuminexDataRow existingRow : wellGroupEntry.getValue())
145+
{
146+
// keep track of well, FI, and FI Bkgd values from existing raw data rows to use in calculating summary stats for the new summary row
147+
wells.add(existingRow.getWell());
148+
if (existingRow.getFi() != null)
149+
fis.add(existingRow.getFi());
150+
if (existingRow.getFiBackground() != null)
151+
fiBkgds.add(existingRow.getFiBackground());
152+
153+
// clone the following properties from the existing row to the newRow:
154+
// extraProperties, container, protocolid, description, wellrole, extraSpecimenInfo,
155+
// specimenID, participantID, visitID, date, dilution, tittration, singlepointcontrol
156+
// note: don't clone rowid, beadcount, lsid
157+
newRow._setExtraProperties(existingRow._getExtraProperties());
158+
newRow.setWellRole(existingRow.getWellRole());
159+
newRow.setContainer(existingRow.getContainer());
160+
newRow.setProtocol(existingRow.getProtocol());
161+
newRow.setDescription(existingRow.getDescription());
162+
newRow.setSpecimenID(existingRow.getSpecimenID());
163+
newRow.setParticipantID(existingRow.getParticipantID());
164+
newRow.setVisitID(existingRow.getVisitID());
165+
newRow.setDate(existingRow.getDate());
166+
newRow.setExtraSpecimenInfo(existingRow.getExtraSpecimenInfo());
167+
newRow.setDilution(existingRow.getDilution());
168+
newRow.setTitration(existingRow.getTitration());
169+
newRow.setSinglePointControl(existingRow.getSinglePointControl());
170+
171+
// we can clone stdev and cv from existing raw rows because LuminexDataHandler ensureSummaryStats() calculates them
172+
newRow.setStdDev(existingRow.getStdDev());
173+
newRow.setCv(existingRow.getCv());
174+
}
175+
176+
// Calculate FI and FI-BKGD values for the new summary row based on the existing raw data rows with the same dataId, analyteId, type, and standard.
177+
// similar to LuminexDataHandler ensureSummaryStats()
178+
if (!fis.isEmpty())
179+
{
180+
MathStat statsFi = service.getStats(ArrayUtils.toPrimitive(fis.toArray(new Double[0])));
181+
newRow.setFi(Math.abs(statsFi.getMean()));
182+
newRow.setFiString(newRow.getFi().toString());
183+
}
184+
if (!fiBkgds.isEmpty())
185+
{
186+
MathStat statsFiBkgd = service.getStats(ArrayUtils.toPrimitive(fiBkgds.toArray(new Double[0])));
187+
newRow.setFiBackground(Math.abs(statsFiBkgd.getMean()));
188+
newRow.setFiBackgroundString(newRow.getFiBackground().toString());
189+
}
190+
191+
// Calculate well to be a comma-separated list of wells from the existing raw data rows
192+
newRow.setWell(String.join(",", wells));
193+
194+
// Generate an LSID for the new summary row
195+
Lsid.LsidBuilder builder = new Lsid.LsidBuilder(LuminexAssayProvider.LUMINEX_DATA_ROW_LSID_PREFIX,"");
196+
newRow.setLsid(builder.setObjectId(GUID.makeGUID()).toString());
197+
198+
// Insert the new summary row into the database.
199+
// similar to LuminexDataHandler saveDataRows()
200+
LuminexImportHelper helper = new LuminexImportHelper();
201+
Map<String, Object> row = new CaseInsensitiveHashMap<>(newRow._getExtraProperties());
202+
ObjectFactory<LuminexDataRow> f = ObjectFactory.Registry.getFactory(LuminexDataRow.class);
203+
row.putAll(f.toMap(newRow, null));
204+
row.put("summary", true); // make sure the extra properties value from the raw row didn't override the summary setting
205+
try
206+
{
207+
OntologyManager.insertTabDelimited(tableInfo, expRun.getContainer(), user, helper, MapDataIterator.of(List.of(row)).getDataIterator(new DataIteratorContext()), true, LOG, null);
208+
String comment = "Inserted missing summary row for Luminex runId: " + runid + ", dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + ", standard: " + groupKey.standard;
209+
ExperimentService.get().auditRunEvent(user, protocol, expRun, null, "LuminexUpgradeCode.checkForMissingSummaryRows: " + comment, null);
210+
LOG.info("..." + comment);
211+
}
212+
catch (BatchValidationException e)
213+
{
214+
LOG.warn("...failed to insert missing summary row for Luminex dataId: " + dataId + ", analyteId: " + analyteId + ", type: " + type + ", standard: " + groupKey.standard, e);
215+
}
216+
}
217+
});
218+
219+
tx.commit();
220+
}
221+
}
222+
}

‎luminex/webapp/luminex/LeveyJenningsTrackingDataPanel.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,11 @@ LABKEY.LeveyJenningsTrackingDataPanel = Ext.extend(Ext.Component, {
104104
filters.push(LABKEY.Filter.create('Titration/IncludeInQcReport', true));
105105
}
106106

107+
var minDateProp = LABKEY.getModuleContext("luminex")?.leveyJenningsMinDate;
108+
if (minDateProp) {
109+
filters.push(LABKEY.Filter.create('Analyte/Data/AcquisitionDate', minDateProp, LABKEY.Filter.Types.GTE));
110+
}
111+
107112
var buttonBarItems = [
108113
LABKEY.QueryWebPart.standardButtons.views,
109114
LABKEY.QueryWebPart.standardButtons.exportRows,

‎luminex/webapp/luminex/LeveyJenningsTrendPlotPanel.js‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -154,13 +154,9 @@ LABKEY.LeveyJenningsTrendPlotPanel = Ext.extend(Ext.FormPanel, {
154154
});
155155
this.items.push(this.trendTabPanel);
156156

157-
this.fbar = [
158-
{xtype: 'label', text: 'The default plot is showing the most recent ' + this.defaultRowSize + ' data points.'},
159-
];
157+
this.fbar = [];
160158

161159
LABKEY.LeveyJenningsTrendPlotPanel.superclass.initComponent.call(this);
162-
163-
this.fbar.hide();
164160
},
165161

166162
// function called by the JSP when the graph params are selected and the "Apply" button is clicked
@@ -193,10 +189,23 @@ LABKEY.LeveyJenningsTrendPlotPanel = Ext.extend(Ext.FormPanel, {
193189
}
194190

195191
this.plotDataLoadComplete = true;
196-
this.fbar.setVisible(!hasReportFilter && this.trendDataStore.getTotalCount() >= this.defaultRowSize);
192+
193+
this.fbar.removeAll();
194+
if (!hasReportFilter && this.trendDataStore.getTotalCount() >= this.defaultRowSize) {
195+
this.fbar.add({xtype: 'label', text: 'The default plot is showing the most recent ' + this.defaultRowSize + ' data points.'});
196+
}
197+
if (this.getMinDateModuleProp()) {
198+
this.fbar.add({xtype: 'label', text: 'Filtering for acquisition date greater than or equal to ' + this.getMinDateModuleProp() + '.'});
199+
}
200+
this.fbar.doLayout();
201+
197202
this.updateTrendPlot();
198203
},
199204

205+
getMinDateModuleProp: function() {
206+
return LABKEY.getModuleContext("luminex")?.leveyJenningsMinDate;
207+
},
208+
200209
setTrendPlotLoading: function() {
201210
this.plotDataLoadComplete = false;
202211
var plotType = this.trendTabPanel.getActiveTab().itemId;

0 commit comments

Comments
 (0)