merge latest from main

config/api_list.yaml

@@ -2,7 +2,7 @@
 # APIs in this list are accessible by the main endpoints v1/query and v1/asyncquery
 include:
  - id: d22b657426375a5295e7da8a303b9893
- name: BioLink API
+ name: Monarch API
  - id: 0212611d1c670f9107baf00b77f0889a
  name: CTD API
  primarySource: true
@@ -36,6 +36,10 @@ include:
  name: BioThings DISEASES API
  - id: 1f47552dabd67351d4c625adb0a10d00
  name: BioThings EBIgene2phenotype API
+ - id: f1b8f64c316a01d1722f0fb842499fe5
+ name: BioThings FooDB API
+ - id: 895ec14a3650ec7ad85959a2d1554e2f
+ name: BioThings FoodData Central API
  - id: cc857d5b7c8b7609b5bbb38ff990bfff
  name: BioThings GO Biological Process API
  - id: f339b28426e7bf72028f60feefcd7465
@@ -56,6 +60,8 @@ include:
  name: BioThings PFOCR API
  - id: b772ebfbfa536bba37764d7fddb11d6f
  name: BioThings RARe-SOURCE API
+ - id: 1138c3297e8e403b6ac10cff5609b319
+ name: BioThings repoDB API
  - id: 03283cc2b21c077be6794e1704b1d230
  name: BioThings Rhea API
  - id: 1d288b3a3caf75d541ffaae3aab386c8

config/smartapi_overrides.yaml

@@ -2,8 +2,10 @@
 config:
  only_overrides: false
 apis:
- 671b45c0301c8624abbd26ae78449ca2: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/orphanet-change/mydisease.info/smartapi.yaml
- b772ebfbfa536bba37764d7fddb11d6f: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/orphanet-change/ncats_rare_source/smartapi.yaml
- 326eb1e437303bee27d3cef29227125d: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/orphanet-change/complexportal/smartapi.yaml
- 0212611d1c670f9107baf00b77f0889a: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/ctd-batch-query/CTD/smartapi.yaml
- 8f08d1446e0bb9c2b323713ce83e2bd3: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/mychem-fda-orphan-edit/mychem.info/openapi_full.yml
+ 32f36164fabed5d3abe6c2fd899c9418: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/idisk/smartapi.yaml
+ b48c34df08d16311e3bca06b135b828d: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/suppkg/suppkg.yaml
+ e481efd21f8e8c1deac05662439c2294: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/ttd/smartapi.yaml
+ 1d288b3a3caf75d541ffaae3aab386c8: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/semmeddb/smartapi.yaml
+ 8f08d1446e0bb9c2b323713ce83e2bd3: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/mychem.info/openapi_full.yml
+ 1138c3297e8e403b6ac10cff5609b319: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/repodb/smartapi.yaml
+ 978fe380a147a8641caf72320862697b: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/treats-refactor-tmkp/text_mining/smartapi.yaml

docs/smartapi.yaml

@@ -12,7 +12,7 @@ info:
  component: ARA
  team:
  - Exploring Agent
- biolink-version: "3.5.0"
+ biolink-version: "4.1.6"
  infores: "infores:biothings-explorer"
  x-trapi:
  version: 1.4.0

src/controllers/cron/update_local_smartapi.ts

@@ -124,6 +124,21 @@ function getServerFromSpec(spec: Spec): string {
  return sorted_servers[0].url;
 }
 
+function sortObject(object: unknown) {
+ if (Array.isArray(object)) {
+ return object.sort();
+ }
+
+ // apparently typeof null is object
+ if (typeof object === 'object' && object !== null) {
+ return Object.keys(object).sort().reduce((acc, key) => {
+ acc[key] = sortObject(object[key]);
+ return acc;
+ }, {});
+ }
+
+ return object;
+}
 function getTRAPIWithPredicatesEndpoint(specs: Spec[]): API[] {
  const trapi: API[] = [];
  let excluded_list = SMARTAPI_EXCLUSIONS.map(api => api.id);
@@ -255,12 +270,13 @@ async function getOpsFromEndpoint(metadata) {
 
 async function getOpsFromPredicatesEndpoints(specs: Spec[]) {
  const metadatas = getTRAPIWithPredicatesEndpoint(specs);
+ metadatas.sort((a, b) => a.association.smartapi.id.localeCompare(b.association.smartapi.id));
  let res = [];
  debug(`Now caching predicates from ${metadatas.length} TRAPI APIs`);
  await Promise.allSettled(metadatas.map(metadata => getOpsFromEndpoint(metadata))).then(results => {
  results.map(rec => {
  if (rec.status === "fulfilled" && rec.value) {
- res.push(rec.value);
+ res.push(sortObject(rec.value));
  }
  });
  });

src/controllers/threading/taskHandler.ts

@@ -16,7 +16,7 @@ import * as Sentry from "@sentry/node";
 import { ProfilingIntegration } from "@sentry/profiling-node";
 import OpenTelemetry, { Span } from "@opentelemetry/api";
 import { Telemetry } from "@biothings-explorer/utils";
-import { InnerTaskData, TaskInfo } from "@biothings-explorer/types";
+import { InnerTaskData } from "@biothings-explorer/types";
 
 // use SENTRY_DSN environment variable
 try {
@@ -46,7 +46,14 @@ try {
  debug(error);
 }
 
-async function runTask({ req, route, port, job = { jobId: undefined, queueName: undefined } }: InnerTaskData) {
+async function runTask({
+ req,
+ route,
+ traceparent,
+ tracestate,
+ port,
+ job = { jobId: undefined, queueName: undefined },
+}: InnerTaskData) {
  debug(`Worker thread ${threadId} beginning ${Piscina.workerData.queue} task.`);
 
  global.SCHEMA_VERSION = "1.4.0";
@@ -83,7 +90,13 @@ async function runTask({ req, route, port, job = { jobId: undefined, queueName:
  scope.setSpan(transaction);
  });
 
- span = OpenTelemetry.trace.getTracer("biothings-explorer-thread").startSpan(routeNames[route]);
+ span = OpenTelemetry.trace
+ .getTracer("biothings-explorer-thread")
+ .startSpan(
+ routeNames[route],
+ undefined,
+ OpenTelemetry.propagation.extract(OpenTelemetry.context.active(), { traceparent, tracestate }),
+ );
  span.setAttribute("bte.requestData", JSON.stringify(req.data.queryGraph));
  Telemetry.setOtelSpan(span);
  } catch (error) {

src/controllers/threading/threadHandler.ts

@@ -1,5 +1,6 @@
 import { MessageChannel, threadId } from "worker_threads";
 import Debug from "debug";
+import { context, propagation } from "@opentelemetry/api";
 const debug = Debug("bte:biothings-explorer-trapi:threading");
 import path from "path";
 import { redisClient } from "@biothings-explorer/utils";
@@ -102,7 +103,14 @@ async function queueTaskToWorkers(pool: Piscina, taskInfo: TaskInfo, route: stri
  let workerThreadID: string;
  const abortController = new AbortController();
  const { port1: toWorker, port2: fromWorker } = new MessageChannel();
- const taskData: InnerTaskData = { req: taskInfo, route, port: toWorker };
+
+ // get otel context
+
+ const otelData: Partial<{ traceparent: string; tracestate: string }> = {};
+ propagation.inject(context.active(), otelData);
+ const { traceparent, tracestate } = otelData;
+
+ const taskData: InnerTaskData = { req: taskInfo, route, traceparent, tracestate, port: toWorker };
 
  // Propagate data between task runner and bull job
  if (job) taskData.job = { jobId: job.id, queueName: job.queue.name };