Skip to content

Commit 5b44522

Browse files
committed
Merge branch 'develop' into TASK-5564
2 parents 0066252 + a429a6e commit 5b44522

File tree

6 files changed

+636
-23
lines changed

6 files changed

+636
-23
lines changed

.github/workflows/debug-token.yml

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
name: Debug ZETTA_REPO_ACCESS_TOKEN and Permissions
2+
3+
on:
4+
workflow_dispatch:
5+
6+
jobs:
7+
debug-token-and-permissions:
8+
runs-on: ${{ vars.UBUNTU_VERSION }}
9+
env:
10+
ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
11+
steps:
12+
- uses: actions/checkout@v4
13+
with:
14+
fetch-depth: 1
15+
- name: Show runner and workflow context
16+
run: |
17+
echo "Runner: $(uname -a)"
18+
echo "GitHub Actor: $GITHUB_ACTOR"
19+
echo "Workflow: $GITHUB_WORKFLOW"
20+
echo "Event: $GITHUB_EVENT_NAME"
21+
echo "Repository: $GITHUB_REPOSITORY"
22+
echo "Job: $GITHUB_JOB"
23+
echo "Workspace: $GITHUB_WORKSPACE"
24+
echo "Home: $HOME"
25+
echo "Shell: $SHELL"
26+
echo "GitHub Ref: $GITHUB_REF"
27+
echo "GitHub SHA: $GITHUB_SHA"
28+
echo "GitHub Run ID: $GITHUB_RUN_ID"
29+
echo "GitHub Run Number: $GITHUB_RUN_NUMBER"
30+
31+
- name: Show GitHub Actions permissions
32+
run: |
33+
echo "Permissions JSON:"
34+
cat $GITHUB_EVENT_PATH | jq '.workflow_run?.permissions // .permissions // "No permissions found"'
35+
36+
- name: Check ZETTA_REPO_ACCESS_TOKEN presence and length
37+
run: |
38+
if [ -z "$ZETTA_REPO_ACCESS_TOKEN" ]; then
39+
echo "ZETTA_REPO_ACCESS_TOKEN is NOT set"
40+
exit 1
41+
else
42+
echo "ZETTA_REPO_ACCESS_TOKEN is set"
43+
echo "Length: ${#ZETTA_REPO_ACCESS_TOKEN}"
44+
echo "First 4 chars: ${ZETTA_REPO_ACCESS_TOKEN:0:4}"
45+
fi
46+
- id: get_ci_core_xetabase_branch
47+
name: "Get CI-CORE current branch for Xetabase from target branch"
48+
run: |
49+
chmod +x ./.github/workflows/scripts/get_opencga_enterprise_branch.sh
50+
echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}"
51+
xetabase_branch=$(./.github/workflows/scripts/get_opencga_enterprise_branch.sh "java-common-libs" "develop" "TASK-8067")
52+
echo "__CI CORE Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
53+
REPO_URI="https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git"
54+
echo "$(git ls-remote "$REPO_URI" "TASK-8067")"
55+
env:
56+
ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
57+
- id: get_xetabase_branch
58+
name: "Get JCL current branch for Xetabase from target branch"
59+
run: |
60+
chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh
61+
echo "github.event.pull_request.base.ref: develop"
62+
echo "github.event.pull_request.head.ref: TASK-8067"
63+
echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}"
64+
xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "develop" "TASK-8067")
65+
echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
66+
env:
67+
ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
68+
- name: Clone OpenCGA Enterprise branch '${{ inputs.branch }}'
69+
uses: actions/checkout@v4
70+
with:
71+
repository: zetta-genomics/opencga-enterprise
72+
ref: TASK-8067
73+
token: ${{ env.ZETTA_REPO_ACCESS_TOKEN }}
74+
path: xetbase-opencga-enterprise
75+
fetch-depth: "10"
76+
77+
- name: Try to clone the private repo with token
78+
run: |
79+
set -x
80+
git clone https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git || true
81+
82+
- name: Show authenticated user (if possible)
83+
run: |
84+
curl -H "Authorization: token $ZETTA_REPO_ACCESS_TOKEN" https://api.github.com/user
85+
86+
- name: Show environment variables (filtered)
87+
run: |
88+
env | grep -E 'GITHUB|ZETTA|RUNNER|CI'
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#!/usr/bin/env bash
2+
# get_opencga_enterprise_branch.sh
3+
#
4+
# This script computes the correct opencga-enterprise branch to use for testing a PR in any Xetabase component repo.
5+
# It unifies the logic previously duplicated in java-common-libs, biodata, and opencga.
6+
#
7+
# Inputs:
8+
# 1. project (artifactId from pom.xml)
9+
# 2. base_ref (target branch of the PR)
10+
# 3. head_ref (source branch of the PR)
11+
#
12+
# Output:
13+
# Prints the resolved opencga-enterprise branch to stdout.
14+
# Exits with non-zero and error message if it cannot resolve a branch.
15+
16+
set -euo pipefail
17+
18+
project="$1"
19+
base_ref="$2"
20+
head_ref="$3"
21+
REPO_URI="https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git"
22+
23+
# 1. If the branch begins with 'TASK' and exists in the opencga-enterprise repository, return it
24+
if [[ $head_ref == TASK* ]]; then
25+
if [ "$(git ls-remote "$REPO_URI" "$head_ref")" ] ; then
26+
echo "$head_ref";
27+
exit 0
28+
fi
29+
fi
30+
31+
# 2. If the base_ref is 'develop', always map to develop
32+
if [[ "$base_ref" == "develop" ]]; then
33+
echo "develop"
34+
exit 0
35+
fi
36+
37+
# 3. release-* branch logic
38+
if [[ "$base_ref" =~ ^release-([0-9]+)\. ]]; then
39+
major="${BASH_REMATCH[1]}"
40+
# Project-specific offset for release branch mapping
41+
case "$project" in
42+
java-common-libs)
43+
offset=3
44+
;;
45+
biodata|opencga)
46+
offset=1
47+
;;
48+
opencga-enterprise)
49+
# If the project is opencga-enterprise, use the branch as-is
50+
echo "$base_ref"
51+
exit 0
52+
;;
53+
*)
54+
echo "ERROR: Unknown project '$project' for release branch mapping" >&2
55+
exit 1
56+
;;
57+
esac
58+
new_major=$((major - offset))
59+
if (( new_major < 1 )); then
60+
echo "ERROR: Computed release branch version < 1 for $project (base_ref: $base_ref, offset: $offset)" >&2
61+
exit 1
62+
fi
63+
# Extract the rest of the version (minor and patch) from base_ref
64+
rest_version=$(echo "$base_ref" | sed -E "s/^release-[0-9]+(\..*)/\1/")
65+
target_branch="release-${new_major}${rest_version}"
66+
# Check if the exact branch exists (fix: do not escape quotes)
67+
if [ "$(git ls-remote --heads "$REPO_URI" "$target_branch")" ]; then
68+
echo "$target_branch"
69+
exit 0
70+
else
71+
echo "ERROR: No matching release branch '$target_branch' found in opencga-enterprise for $project (base_ref: $base_ref, offset: $offset)" >&2
72+
exit 1
73+
fi
74+
fi
75+
76+
# 4. Fallback: fail with clear error
77+
echo "ERROR: Could not resolve opencga-enterprise branch for project '$project' (base_ref: $base_ref, head_ref: $head_ref)" >&2
78+
exit 1
79+

commons-datastore/commons-datastore-mongodb/src/main/java/org/opencb/commons/datastore/mongodb/MongoDBCollection.java

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,123 @@ public DataResult update(ClientSession clientSession, List<? extends Bson> queri
450450
start);
451451
}
452452

453+
/**
454+
* Update documents using an aggregation pipeline.
455+
*
456+
* @param query the filter to select the documents to update
457+
* @param pipeline the aggregation pipeline to apply for the update
458+
* @param options additional options for the query and update operation
459+
* @return a DataResult containing the result of the update operation
460+
*/
461+
public DataResult updateWithPipeline(Bson query, List<? extends Bson> pipeline, QueryOptions options) {
462+
return updateWithPipeline(null, query, pipeline, options);
463+
}
464+
465+
/**
466+
* Update documents using an aggregation pipeline.
467+
*
468+
* @param clientSession the client session to use for the operation, or null if not using sessions
469+
* @param query the filter to select the documents to update
470+
* @param pipeline the aggregation pipeline to apply for the update
471+
* @param options additional options for the query and update operation
472+
* @return a DataResult containing the result of the update operation
473+
*/
474+
public DataResult updateWithPipeline(ClientSession clientSession, Bson query, List<? extends Bson> pipeline, QueryOptions options) {
475+
long start = startQuery();
476+
477+
boolean upsert = false;
478+
boolean multi = false;
479+
if (options != null) {
480+
upsert = options.getBoolean(UPSERT);
481+
multi = options.getBoolean(MULTI);
482+
}
483+
484+
UpdateResult updateResult = mongoDBNativeQuery.updateWithPipeline(clientSession, query, pipeline, upsert, multi);
485+
return endWrite(updateResult.getMatchedCount(), updateResult.getUpsertedId() != null ? 1 : 0,
486+
updateResult.getUpsertedId() == null ? updateResult.getModifiedCount() : 0, 0, 0, start);
487+
}
488+
489+
/**
490+
* Finds a document matching the given query, applies an aggregation pipeline to update it, and returns the updated document.
491+
*
492+
* @param query the filter to select the document to update
493+
* @param projection the fields to return in the resulting document
494+
* @param sort the sort criteria to apply before finding the document
495+
* @param pipeline the aggregation pipeline to apply for the update
496+
* @param options additional options for the query and update operation
497+
* @return a DataResult containing the updated document, or an empty result if no document matched
498+
*/
499+
public DataResult<Document> findAndUpdateWithPipeline(Bson query, Bson projection, Bson sort,
500+
List<? extends Bson> pipeline, QueryOptions options) {
501+
return privateFindAndUpdateWithPipeline(null, query, projection, sort, pipeline, options, null, null);
502+
}
503+
504+
/**
505+
* Finds a document matching the given query, applies an aggregation pipeline to update it, and returns the updated document.
506+
*
507+
* @param clientSession the client session to use for the operation, or null if not using sessions
508+
* @param query the filter to select the document to update
509+
* @param projection the fields to return in the resulting document
510+
* @param sort the sort criteria to apply before finding the document
511+
* @param pipeline the aggregation pipeline to apply for the update
512+
* @param options additional options for the query and update operation
513+
* @return a DataResult containing the updated document, or an empty result if no document matched
514+
*/
515+
public DataResult<Document> findAndUpdateWithPipeline(ClientSession clientSession, Bson query, Bson projection, Bson sort,
516+
List<? extends Bson> pipeline, QueryOptions options) {
517+
return privateFindAndUpdateWithPipeline(clientSession, query, projection, sort, pipeline, options, null, null);
518+
}
519+
520+
/**
521+
* Finds a document matching the given query, applies an aggregation pipeline to update it, and returns the updated document.
522+
*
523+
* @param query the filter to select the document to update
524+
* @param projection the fields to return in the resulting document
525+
* @param sort the sort criteria to apply before finding the document
526+
* @param pipeline the aggregation pipeline to apply for the update
527+
* @param clazz the class type to convert the result to; if null or Document.class, returns a Document
528+
* @param options additional options for the query and update operation
529+
* @param <T> the type of the returned result
530+
* @return a DataResult containing the updated document, or an empty result if no document matched
531+
*/
532+
public <T> DataResult<T> findAndUpdateWithPipeline(Bson query, Bson projection, Bson sort,
533+
List<? extends Bson> pipeline, Class<T> clazz, QueryOptions options) {
534+
return privateFindAndUpdateWithPipeline(null, query, projection, sort, pipeline, options, clazz, null);
535+
}
536+
537+
/**
538+
* Finds a document matching the given query, applies an aggregation pipeline to update it, and returns the updated document.
539+
*
540+
* @param clientSession the client session to use for the operation, or null if not using sessions
541+
* @param query the filter to select the document to update
542+
* @param projection the fields to return in the resulting document
543+
* @param sort the sort criteria to apply before finding the document
544+
* @param pipeline the aggregation pipeline to apply for the update
545+
* @param clazz the class type to convert the result to; if null or Document.class, returns a Document
546+
* @param options additional options for the query and update operation
547+
* @param <T> the type of the returned result
548+
* @return a DataResult containing the updated document, or an empty result if no document matched
549+
*/
550+
public <T> DataResult<T> findAndUpdateWithPipeline(ClientSession clientSession, Bson query, Bson projection, Bson sort,
551+
List<? extends Bson> pipeline, Class<T> clazz, QueryOptions options) {
552+
return privateFindAndUpdateWithPipeline(clientSession, query, projection, sort, pipeline, options, clazz, null);
553+
}
554+
555+
private <T> DataResult<T> privateFindAndUpdateWithPipeline(ClientSession clientSession, Bson query, Bson projection, Bson sort,
556+
List<? extends Bson> pipeline, QueryOptions options, Class<T> clazz,
557+
ComplexTypeConverter<T, Document> converter) {
558+
long start = startQuery();
559+
Document result = mongoDBNativeQuery.findAndUpdateWithPipeline(clientSession, query, projection, sort, pipeline, options);
560+
if (clazz != null && !clazz.equals(Document.class)) {
561+
try {
562+
return endQuery(Collections.singletonList(objectMapper.readValue(objectWriter.writeValueAsString(result), clazz)), start);
563+
} catch (IOException e) {
564+
logger.error("Error deserializing result: " + e.getMessage(), e);
565+
}
566+
}
567+
return endQuery(Collections.singletonList(result), start);
568+
}
569+
453570
public DataResult remove(Bson query, QueryOptions options) {
454571
return remove(null, query, options);
455572
}

commons-datastore/commons-datastore-mongodb/src/main/java/org/opencb/commons/datastore/mongodb/MongoDBNativeQuery.java

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,99 @@ public BulkWriteResult update(ClientSession clientSession, List<? extends Bson>
461461
}
462462
}
463463

464+
/**
465+
* Update documents using an aggregation pipeline.
466+
*
467+
* @param query The filter to select the documents to update.
468+
* @param pipeline The aggregation pipeline specifying the update operations.
469+
* @param upsert Whether to insert a new document if no documents match the query.
470+
* @param multi Whether to update multiple documents or just one.
471+
* @return The result of the update operation.
472+
*/
473+
public UpdateResult updateWithPipeline(Bson query, List<? extends Bson> pipeline, boolean upsert, boolean multi) {
474+
return updateWithPipeline(null, query, pipeline, upsert, multi);
475+
}
476+
477+
/**
478+
* Update documents using an aggregation pipeline.
479+
*
480+
* @param clientSession Session in which the operation will be performed. Can be null.
481+
* @param query The filter to select the documents to update.
482+
* @param pipeline The aggregation pipeline specifying the update operations.
483+
* @param upsert Whether to insert a new document if no documents match the query.
484+
* @param multi Whether to update multiple documents or just one.
485+
* @return The result of the update operation.
486+
*/
487+
public UpdateResult updateWithPipeline(ClientSession clientSession, Bson query, List<? extends Bson> pipeline,
488+
boolean upsert, boolean multi) {
489+
UpdateOptions updateOptions = new UpdateOptions().upsert(upsert);
490+
if (multi) {
491+
if (clientSession != null) {
492+
return dbCollection.updateMany(clientSession, query, pipeline, updateOptions);
493+
} else {
494+
return dbCollection.updateMany(query, pipeline, updateOptions);
495+
}
496+
} else {
497+
if (clientSession != null) {
498+
return dbCollection.updateOne(clientSession, query, pipeline, updateOptions);
499+
} else {
500+
return dbCollection.updateOne(query, pipeline, updateOptions);
501+
}
502+
}
503+
}
504+
505+
/**
506+
* Finds and updates a single document using an aggregation pipeline.
507+
*
508+
* @param query The filter to select the document to update.
509+
* @param projection The fields to return in the resulting document.
510+
* @param sort The sort criteria to apply before updating.
511+
* @param pipeline The aggregation pipeline specifying the update operations.
512+
* @param options Additional options such as upsert and returnNew.
513+
* @return The updated document, or null if no document matched the query.
514+
*/
515+
public Document findAndUpdateWithPipeline(Bson query, Bson projection, Bson sort,
516+
List<? extends Bson> pipeline, QueryOptions options) {
517+
return findAndUpdateWithPipeline(null, query, projection, sort, pipeline, options);
518+
}
519+
520+
/**
521+
* Finds and updates a single document using an aggregation pipeline.
522+
*
523+
* @param clientSession Session in which the operation will be performed. Can be null.
524+
* @param query The filter to select the document to update.
525+
* @param projection The fields to return in the resulting document.
526+
* @param sort The sort criteria to apply before updating.
527+
* @param pipeline The aggregation pipeline specifying the update operations.
528+
* @param options Additional options such as upsert and returnNew.
529+
* @return The updated document, or null if no document matched the query.
530+
*/
531+
public Document findAndUpdateWithPipeline(ClientSession clientSession, Bson query, Bson projection, Bson sort,
532+
List<? extends Bson> pipeline, QueryOptions options) {
533+
boolean upsert = false;
534+
boolean returnNew = false;
535+
536+
if (options != null) {
537+
if (projection == null) {
538+
projection = getProjection(projection, options);
539+
}
540+
upsert = options.getBoolean("upsert", false);
541+
returnNew = options.getBoolean("returnNew", false);
542+
}
543+
544+
FindOneAndUpdateOptions findOneAndUpdateOptions = new FindOneAndUpdateOptions()
545+
.sort(sort)
546+
.projection(projection)
547+
.upsert(upsert)
548+
.returnDocument(returnNew ? ReturnDocument.AFTER : ReturnDocument.BEFORE);
549+
550+
if (clientSession != null) {
551+
return dbCollection.findOneAndUpdate(clientSession, query, pipeline, findOneAndUpdateOptions);
552+
} else {
553+
return dbCollection.findOneAndUpdate(query, pipeline, findOneAndUpdateOptions);
554+
}
555+
}
556+
464557
private IndexOutOfBoundsException wrongQueryUpdateSize(List<? extends Bson> queries, List<? extends Bson> updates) {
465558
return new IndexOutOfBoundsException("QueryList.size=" + queries.size()
466559
+ " and UpdatesList.size=" + updates.size() + " must be the same size.");

0 commit comments

Comments
 (0)