From 7cd9abe8dd624240ef866a1ff254532bf5122b92 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Mon, 8 Sep 2025 12:37:42 -0700 Subject: [PATCH 1/8] Adding an configuration section and template file for dataset external links. --- datasetExternalLinkTemplates.example.json | 14 ++++++++++++++ src/config/configuration.ts | 5 +++++ 2 files changed, 19 insertions(+) create mode 100644 datasetExternalLinkTemplates.example.json diff --git a/datasetExternalLinkTemplates.example.json b/datasetExternalLinkTemplates.example.json new file mode 100644 index 000000000..3ee23320f --- /dev/null +++ b/datasetExternalLinkTemplates.example.json @@ -0,0 +1,14 @@ +[ + { + "title": "Franzviewer II", + "url_template": "https://franz.site.com/franzviewer?id=${dataset.pid}", + "description_template": "View ${dataset.numberOfFiles} files in Franz' own personal viewer", + "filter": "(dataset.type == 'derived') && dataset.owner.includes('Franz')" + }, + { + "title": "High Beam-Energy View", + "url_template": "https://beamviewer.beamline.net/highenergy?id=${dataset.pid}", + "description_template": "The high-energy beamviewer (value ${dataset.scientificMetadata?.beamEnergy?.value}) at beamCo", + "filter": "(dataset.scientificMetadata?.beamEnergy?.value > 20)" + } +] diff --git a/src/config/configuration.ts b/src/config/configuration.ts index 68cde662b..6d5d94db8 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -71,6 +71,7 @@ const configuration = () => { }; const jsonConfigMap: { [key: string]: object | object[] | boolean } = { datasetTypes: {}, + datasetExternalLinkTemplates: [], proposalTypes: {}, }; const jsonConfigFileList: { [key: string]: string } = { @@ -80,6 +81,9 @@ const configuration = () => { process.env.FRONTEND_THEME_FILE || "./src/config/frontend.theme.json", loggers: process.env.LOGGERS_CONFIG_FILE || "loggers.json", datasetTypes: process.env.DATASET_TYPES_FILE || "datasetTypes.json", + datasetExternalLinkTemplates: + process.env.DATASET_EXTERNAL_LINK_TEMPLATES_FILE || + "datasetExternalLinkTemplates.json", proposalTypes: process.env.PROPOSAL_TYPES_FILE || "proposalTypes.json", metricsConfig: process.env.METRICS_CONFIG_FILE || "metricsConfig.json", publishedDataConfig: @@ -402,6 +406,7 @@ const configuration = () => { policyRetentionShiftInYears: process.env.POLICY_RETENTION_SHIFT ?? -1, }, datasetTypes: jsonConfigMap.datasetTypes, + datasetExternalLinkTemplates: jsonConfigMap.datasetExternalLinkTemplates, proposalTypes: jsonConfigMap.proposalTypes, frontendConfig: jsonConfigMap.frontendConfig, frontendTheme: jsonConfigMap.frontendTheme, From a5ac1cc8dbccd745282a2d6577b202a28bcafd1e Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Mon, 8 Sep 2025 12:39:05 -0700 Subject: [PATCH 2/8] Minor edit: Untangling some logic. --- src/datasets/datasets.controller.ts | 66 +++++++++++++---------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index 84897e79a..bbec881a6 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -912,9 +912,9 @@ export class DatasetsController { outputDatasets = datasets.map((dataset) => this.convertCurrentToObsoleteSchema(dataset), ); - await Promise.all( - outputDatasets.map(async (dataset) => { - if (includeFilters) { + if (includeFilters) { + await Promise.all( + outputDatasets.map(async (dataset) => { await Promise.all( includeFilters.map(async ({ relation }) => { switch (relation) { @@ -946,13 +946,9 @@ export class DatasetsController { } }), ); - } else { - /* eslint-disable @typescript-eslint/no-unused-expressions */ - // TODO: check the eslint error "Expected an assignment or function call and instead saw an expression" - dataset; - } - }), - ); + }), + ); + } } return outputDatasets as OutputDatasetObsoleteDto[]; } @@ -1243,35 +1239,31 @@ export class DatasetsController { if (outputDataset) { const includeFilters = mergedFilters.include ?? []; - await Promise.all( - includeFilters.map(async ({ relation }) => { - switch (relation) { - case "attachments": { - outputDataset.attachments = await this.attachmentsService.findAll( - { - where: { - datasetId: outputDataset.pid, + if (includeFilters) { + await Promise.all( + includeFilters.map(async ({ relation }) => { + switch (relation) { + case "attachments": { + outputDataset.attachments = await this.attachmentsService.findAll( + { + where: { + datasetId: outputDataset.pid, + }, }, - }, - ); - break; - } - case "origdatablocks": { - outputDataset.origdatablocks = - await this.origDatablocksService.findAll({ - where: { datasetId: outputDataset.pid }, - }); - break; + ); + break; + } + case "origdatablocks": { + outputDataset.origdatablocks = + await this.origDatablocksService.findAll({ + where: { datasetId: outputDataset.pid }, + }); + break; + } } - case "datablocks": { - outputDataset.datablocks = await this.datablocksService.findAll({ - where: { datasetId: outputDataset.pid }, - }); - break; - } - } - }), - ); + }) + ); + } } return outputDataset; } From ae07731aa666c41d084bdd72548aa803872138d7 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Mon, 8 Sep 2025 12:39:18 -0700 Subject: [PATCH 3/8] Slightly better phrasing. --- src/datasets/datasets.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index bbec881a6..bf4c44751 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -1204,7 +1204,7 @@ export class DatasetsController { @ApiOperation({ summary: "It returns the first dataset found.", description: - "It returns the first dataset of the ones that matches the filter provided. The list returned can be modified by providing a filter.", + "Returns the first dataset that matches the provided filters.", }) @ApiQuery({ name: "filter", From c403b88be89bc4881753bdb44c42cc535ade64cd Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Mon, 8 Sep 2025 12:40:34 -0700 Subject: [PATCH 4/8] Endpoints for v3 and v4 to calculate and return external links for the dataset with the given id. --- src/datasets/datasets.controller.ts | 39 ++++++++++++++++ src/datasets/datasets.service.ts | 54 ++++++++++++++++++++++ src/datasets/datasets.v4.controller.ts | 41 ++++++++++++++++ src/datasets/schemas/externallink.class.ts | 32 +++++++++++++ 4 files changed, 166 insertions(+) create mode 100644 src/datasets/schemas/externallink.class.ts diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index bf4c44751..3f700e6bb 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -77,6 +77,7 @@ import { PartialUpdateDatablockDto } from "src/datablocks/dto/update-datablock.d import { Datablock } from "src/datablocks/schemas/datablock.schema"; import { LogbooksService } from "src/logbooks/logbooks.service"; import { Logbook } from "src/logbooks/schemas/logbook.schema"; +import { ExternalLinkClass } from "./schemas/externallink.class"; import { CreateDatasetOrigDatablockDto } from "src/origdatablocks/dto/create-dataset-origdatablock"; import { CreateOrigDatablockDto } from "src/origdatablocks/dto/create-origdatablock.dto"; import { UpdateOrigDatablockDto } from "src/origdatablocks/dto/update-origdatablock.dto"; @@ -1815,6 +1816,44 @@ export class DatasetsController { return await this.convertCurrentToObsoleteSchema(outputDatasetDto); } + // GET /datasets/:id/externallinks + @UseGuards(PoliciesGuard) + @CheckPolicies("datasets", (ability: AppAbility) => + ability.can(Action.DatasetRead, DatasetClass) || + ability.can(Action.DatasetReadOnePublic, DatasetClass), + ) + @Get("/:pid/externallinks") + @ApiOperation({ + summary: "Returns dataset external links.", + description: + "Returns the applicable external links for the dataset with the given pid.", + }) + @ApiParam({ + name: "pid", + description: "Id of the dataset to return external links", + type: String, + }) + @ApiResponse({ + status: HttpStatus.OK, + type: ExternalLinkClass, + isArray: true, + description: "A list of exernal link objects.", + }) + async findExternalLinksById( + @Req() request: Request, + @Param("pid") id: string, + ) { + const links = await this.datasetsService.findExternalLinksById(id); + + await this.checkPermissionsForDatasetExtended( + request, + id, + Action.DatasetRead, + ); + + return links; + } + // GET /datasets/:id/thumbnail @UseGuards(PoliciesGuard) @CheckPolicies( diff --git a/src/datasets/datasets.service.ts b/src/datasets/datasets.service.ts index ceba48a40..4bec7af2f 100644 --- a/src/datasets/datasets.service.ts +++ b/src/datasets/datasets.service.ts @@ -38,6 +38,7 @@ import { PartialUpdateDatasetWithHistoryDto, UpdateDatasetDto, } from "./dto/update-dataset.dto"; +import { ExternalLinkClass } from "./schemas/externallink.class"; import { IDatasetFields } from "./interfaces/dataset-filters.interface"; import { DatasetClass, DatasetDocument } from "./schemas/dataset.schema"; import { @@ -395,6 +396,59 @@ export class DatasetsService { throw new NotFoundException(error); } } + + async findExternalLinksById( + id: string, + ): Promise { + + const thisDataSet = await this.findOneComplete({ + where: { pid: id }, + include: [DatasetLookupKeysEnum.all] + }); + + if (!thisDataSet) { + // no luck. we need to create a new dataset + throw new NotFoundException(`Dataset #${id} not found`); + } + + interface ExternalLinkTemplateConfig { + title: string; + url_template: string; + description_template: string; + filter: string; + } + + const templates: ExternalLinkTemplateConfig[] | undefined = + this.configService.get("datasetExternalLinkTemplates"); + if (!templates) { + return []; + } + + return templates + .filter((d) => { + const filterFn = new Function( + "dataset", + `return (${d.filter});`, + ); + return filterFn(thisDataSet); + }) + .map((d) => { + const urlFn = new Function( + "dataset", + `return (\`${d.url_template}\`);`, + ); + const descriptionFn = new Function( + "dataset", + `return (\`${d.description_template}\`);`, + ); + return { + url: urlFn(thisDataSet), + title: d.title, + description: descriptionFn(thisDataSet), + }; + }); + } + // Get metadata keys async metadataKeys( filters: IFilters, diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts index ac7cd4813..d6b308d3b 100644 --- a/src/datasets/datasets.v4.controller.ts +++ b/src/datasets/datasets.v4.controller.ts @@ -88,6 +88,7 @@ import { HistoryClass } from "./schemas/history.schema"; import { LifecycleClass } from "./schemas/lifecycle.schema"; import { RelationshipClass } from "./schemas/relationship.schema"; import { TechniqueClass } from "./schemas/technique.schema"; +import { ExternalLinkClass } from "./schemas/externallink.class"; import { isEqual } from "lodash"; @@ -687,6 +688,46 @@ export class DatasetsV4Controller { return this.datasetsService.count(finalFilters); } + + // GET /datasets/:id/externallinks + @UseGuards(PoliciesGuard) + @CheckPolicies("datasets", (ability: AppAbility) => + ability.can(Action.DatasetRead, DatasetClass), + ) + @Get("/:pid/externallinks") + @ApiOperation({ + summary: "Returns dataset external links.", + description: + "Returns the applicable external links for the dataset with the given pid.", + }) + @ApiParam({ + name: "pid", + description: "Id of the dataset to return external links", + type: String, + }) + @ApiResponse({ + status: HttpStatus.OK, + type: ExternalLinkClass, + isArray: true, + description: "A list of exernal link objects.", + }) + async findExternalLinksById( + @Req() request: Request, + @Param("pid") id: string, + ) { + + const links = await this.datasetsService.findExternalLinksById(id); + + await this.checkPermissionsForDatasetExtended( + request, + id, + Action.DatasetRead, + ); + + return links; + } + + // GET /datasets/:id //@UseGuards(PoliciesGuard) @UseGuards(PoliciesGuard) diff --git a/src/datasets/schemas/externallink.class.ts b/src/datasets/schemas/externallink.class.ts new file mode 100644 index 000000000..86222d630 --- /dev/null +++ b/src/datasets/schemas/externallink.class.ts @@ -0,0 +1,32 @@ +import { ApiProperty } from "@nestjs/swagger"; +import { IsString } from "class-validator"; + +// This class defines the externalLinks field in a dataset. +// That field is not represented in the Mongoose data store, +// so there is no equivalent schema representation for it. + +export class ExternalLinkClass { + @ApiProperty({ + type: String, + required: true, + description: "URL of the external link.", + }) + @IsString() + readonly url: string; + + @ApiProperty({ + type: String, + required: true, + description: "Text to display representing the external link.", + }) + @IsString() + readonly title: string; + + @ApiProperty({ + type: String, + required: false, + description: "Description of the link destination.", + }) + @IsString() + readonly description?: string; +} From c0fe37d2cc6e3ec81b703b33464100bbb644f2a1 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Mon, 8 Sep 2025 12:40:56 -0700 Subject: [PATCH 5/8] Updating README to describe new template file. --- README.md | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 72bb7f27a..667eba452 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,10 @@ Thank you for your interest in contributing to our project! 7. _Optional_ Add [loggers.json](#loggers-configuration) file to the root folder and configure multiple loggers. 8. _Optional_ Add [proposalTypes.json](#proposal-types-configuration) file to the root folder and configure the proposal types. 9. _Optional_ Add [datasetTypes.json](#dataset-types-configuration) file to the root folder and configure the dataset types. -10. `npm run start:dev` -11. Go to http://localhost:3000/explorer to get an overview of available endpoints and database schemas. -12. To be able to run the e2e tests with the same setup as in the Github actions you will need to run `npm run prepare:local` and after that run `npm run start:dev`. This will start all needed containers and copy some configuration to the right place. +10. _Optional_ Add [datasetExternalLinkTemplates.json](#dataset-external-link-templates-configuration) file to the root folder and configure the external link types. +11. `npm run start:dev` +12. Go to http://localhost:3000/explorer to get an overview of available endpoints and database schemas. +13. To be able to run the e2e tests with the same setup as in the Github actions you will need to run `npm run prepare:local` and after that run `npm run start:dev`. This will start all needed containers and copy some configuration to the right place. ## Develop in a container using the docker-compose.dev file @@ -57,11 +58,12 @@ Thank you for your interest in contributing to our project! 5. _Optional_ Mount [loggers.json](#loggers-configuration) file to a volume in the container to configure multiple loggers. 6. _Optional_ Mount [proposalTypes.json](#proposal-types-configuration) file to a volume in the container to configure the proposal types. 7. _Optional_ Mount [datasetTypes.json](#dataset-types-configuration) file to a volume in the container to configure the dataset types. -8. _Optional_ Change the container env variables. -9. _Optional_ Create the file test/config/.env.override to override ENV vars that are used when running the tests. -10. Attach to the container. -11. `npm run start:dev` -12. Go to http://localhost:3000/explorer to get an overview of available endpoints and database schemas. +8. _Optional_ Mount [datasetExternalLinkTemplates.json](#dataset-external-link-templates-configuration) file to a volume in the container to configure the external link types. +9. _Optional_ Change the container env variables. +10. _Optional_ Create the file test/config/.env.override to override ENV vars that are used when running the tests. +11. Attach to the container. +12. `npm run start:dev` +13. Go to http://localhost:3000/explorer to get an overview of available endpoints and database schemas. ## Test the app @@ -113,16 +115,26 @@ The `loggers.example.json` file in the root directory showcases the example of c ### Proposal types configuration -Providing a file called _proposalTypes.json_ at the root of the project, locally or in the container, will be automatically loaded into the application configuration service under property called `proposalTypes` and used for validation against proposal creation and update. +If a file called _proposalTypes.json_ is provided at the root of the project, locally or in the container, it will be automatically loaded into the application configuration service under the property `proposalTypes`. + +This content is used for validation against proposal creation and update. -The `proposalTypes.json.example` file in the root directory showcases the example of configuration structure for proposal types. +The file `proposalTypes.example.json` contains an example. ### Dataset types configuration -When providing a file called _datasetTypes.json_ at the root of the project, locally or in the container, it will be automatically loaded into the application configuration service under property called `datasetTypes` and used for validation against dataset creation and update. The types `Raw` and `Derived` are always valid dataset types by default. +If a file called _datasetTypes.json_ is provided at the root of the project, locally or in the container, it will be automatically loaded into the application configuration service under the property `datasetTypes`. The `datasetTypes.example.json` file in the root directory showcases an example of configuration structure for dataset types. +### Dataset external link templates configuration + +If a file called _datasetExternalLinkTemplates.json_ is provided at at the root of the project, locally or in the container, it will be automatically loaded into the application configuration service under the property `datasetExternalLinkTemplates`. + +The content is used to create links to external websites from individual datasets, based on criteria applied to the dataset metadata. + +The file `datasetExternalLinkTemplates.example.json` contains an example. + ### Published data configuration Providing a file called _publishedDataConfig.json_ at the root of the project, locally or in the container, will be automatically loaded into the application configuration service under property called `publishedDataConfig`. It will be used for published data metadata form generation in the frontend and metadata validation in publication and registration of the published data. From 0adb721c0772e7573f474a9bdd6227f76ea23048 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Mon, 8 Sep 2025 13:14:09 -0700 Subject: [PATCH 6/8] Lint. --- src/datasets/datasets.controller.ts | 22 +++++++++++----------- src/datasets/datasets.service.ts | 12 +++--------- src/datasets/datasets.v4.controller.ts | 3 --- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index 3f700e6bb..0ce61119e 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -1204,8 +1204,7 @@ export class DatasetsController { @Get("/findOne") @ApiOperation({ summary: "It returns the first dataset found.", - description: - "Returns the first dataset that matches the provided filters.", + description: "Returns the first dataset that matches the provided filters.", }) @ApiQuery({ name: "filter", @@ -1245,13 +1244,12 @@ export class DatasetsController { includeFilters.map(async ({ relation }) => { switch (relation) { case "attachments": { - outputDataset.attachments = await this.attachmentsService.findAll( - { + outputDataset.attachments = + await this.attachmentsService.findAll({ where: { datasetId: outputDataset.pid, }, - }, - ); + }); break; } case "origdatablocks": { @@ -1259,10 +1257,10 @@ export class DatasetsController { await this.origDatablocksService.findAll({ where: { datasetId: outputDataset.pid }, }); - break; + break; } } - }) + }), ); } } @@ -1818,9 +1816,11 @@ export class DatasetsController { // GET /datasets/:id/externallinks @UseGuards(PoliciesGuard) - @CheckPolicies("datasets", (ability: AppAbility) => - ability.can(Action.DatasetRead, DatasetClass) || - ability.can(Action.DatasetReadOnePublic, DatasetClass), + @CheckPolicies( + "datasets", + (ability: AppAbility) => + ability.can(Action.DatasetRead, DatasetClass) || + ability.can(Action.DatasetReadOnePublic, DatasetClass), ) @Get("/:pid/externallinks") @ApiOperation({ diff --git a/src/datasets/datasets.service.ts b/src/datasets/datasets.service.ts index 4bec7af2f..e5942fd01 100644 --- a/src/datasets/datasets.service.ts +++ b/src/datasets/datasets.service.ts @@ -397,13 +397,10 @@ export class DatasetsService { } } - async findExternalLinksById( - id: string, - ): Promise { - + async findExternalLinksById(id: string): Promise { const thisDataSet = await this.findOneComplete({ where: { pid: id }, - include: [DatasetLookupKeysEnum.all] + include: [DatasetLookupKeysEnum.all], }); if (!thisDataSet) { @@ -426,10 +423,7 @@ export class DatasetsService { return templates .filter((d) => { - const filterFn = new Function( - "dataset", - `return (${d.filter});`, - ); + const filterFn = new Function("dataset", `return (${d.filter});`); return filterFn(thisDataSet); }) .map((d) => { diff --git a/src/datasets/datasets.v4.controller.ts b/src/datasets/datasets.v4.controller.ts index d6b308d3b..c6583e723 100644 --- a/src/datasets/datasets.v4.controller.ts +++ b/src/datasets/datasets.v4.controller.ts @@ -688,7 +688,6 @@ export class DatasetsV4Controller { return this.datasetsService.count(finalFilters); } - // GET /datasets/:id/externallinks @UseGuards(PoliciesGuard) @CheckPolicies("datasets", (ability: AppAbility) => @@ -715,7 +714,6 @@ export class DatasetsV4Controller { @Req() request: Request, @Param("pid") id: string, ) { - const links = await this.datasetsService.findExternalLinksById(id); await this.checkPermissionsForDatasetExtended( @@ -727,7 +725,6 @@ export class DatasetsV4Controller { return links; } - // GET /datasets/:id //@UseGuards(PoliciesGuard) @UseGuards(PoliciesGuard) From 47838feffe64204b0a7c087878510fd6caead9d9 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 5 Nov 2025 14:37:28 -0800 Subject: [PATCH 7/8] More descriptive variable name. --- src/datasets/datasets.service.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/datasets/datasets.service.ts b/src/datasets/datasets.service.ts index 48a9f3efa..e2b3e47a8 100644 --- a/src/datasets/datasets.service.ts +++ b/src/datasets/datasets.service.ts @@ -499,22 +499,22 @@ export class DatasetsService { } return templates - .filter((d) => { - const filterFn = new Function("dataset", `return (${d.filter});`); + .filter((template) => { + const filterFn = new Function("dataset", `return (${template.filter});`); return filterFn(thisDataSet); }) - .map((d) => { + .map((template) => { const urlFn = new Function( "dataset", - `return (\`${d.url_template}\`);`, + `return (\`${template.url_template}\`);`, ); const descriptionFn = new Function( "dataset", - `return (\`${d.description_template}\`);`, + `return (\`${template.description_template}\`);`, ); return { url: urlFn(thisDataSet), - title: d.title, + title: template.title, description: descriptionFn(thisDataSet), }; }); From a28992859028f3869398496ef0152f9a36854659 Mon Sep 17 00:00:00 2001 From: Garrett Birkel Date: Wed, 5 Nov 2025 14:44:01 -0800 Subject: [PATCH 8/8] Lint --- src/datasets/datasets.service.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/datasets/datasets.service.ts b/src/datasets/datasets.service.ts index e2b3e47a8..f0e934f80 100644 --- a/src/datasets/datasets.service.ts +++ b/src/datasets/datasets.service.ts @@ -500,7 +500,10 @@ export class DatasetsService { return templates .filter((template) => { - const filterFn = new Function("dataset", `return (${template.filter});`); + const filterFn = new Function( + "dataset", + `return (${template.filter});`, + ); return filterFn(thisDataSet); }) .map((template) => {