Ldes graphdb connector

README.md

@@ -13,7 +13,7 @@ A Linked Data Event Stream (LDES) is a collection of immutable objects. The HTTP
 ```bash
 npm install
 npm run build
-cd packages/ldes-replicator && npm run start
+npm run start -- -h
 ```
 
 ## Components
@@ -27,6 +27,12 @@ A connector implementation for PostgreSQL.
 **ldes-mongodb-connector:**<br />
 A connector implementation for MongoDB.
 
+**ldes-graphdb-connector:**<br />
+A connector implementation for GraphDB. (can be easily extended to any other datastore compatible with SPARQL)
+
+**ldes-graphdb-version-materialization-connector:**<br />
+Based on the GraphDB connector, for the specific case we only want to keep the last version.
+
 **ldes-dummy-state:**<br />
 An example to write your own state.
 
@@ -60,9 +66,53 @@ Exposes the helm/compose file generators as a Fastify plugin.
 - [Ryan Byloos](https://github.com/ryanbyloos)
 - [Wout Verbiest](https://github.com/woutverbiest)
 
+## Config file
+
+The LDES Replicator is a CLI that takes a JSON file for configuration,
+an example can be found [here](/osoc21/ldes2service/blob/main/config.json.example).
+
+```json
+{
+ // Main settings
+ "replicator": {
+ "ldes": [
+ {
+ "url": "URL to the LDES"
+ },
+ {
+ "url": "URL to the LDES",
+ "shapeUrl": "URL or path of a SHACL shape"
+ }
+ ],
+ "state": {
+ "id": "Used to identify the state in a shared redis server",
+ "host": "hostname",
+ "port": "port number",
+ "password": "password, if needed"
+ },
+ "disable_polling": "If the replicator should stop after the last page of the ldes has been reached",
+ "polling_interval": "delay in ms"
+ },
+ // Connector settings
+ "connectors": {
+ "graphdb": {
+ // NPM package to use as a connector
+ "type": "@ldes/ldes-graphdb-connector",
+ // Connector specific settings cf. Connectors
+ "settings": {
+ ...
+ }
+ }
+ }
+}
+```
+
+Once the config file is created you can add the dynamic dependencies with `npm run start -- -s <config_path>`
+and once it's done, start the LDES replicator with `npm run start -- <config_path>`
+
 ## Running it with Docker
 
-The LDES Replicator is available as a docker image: `ghcr.io/osoc21/ldes-replicator:helm-chart`
+The LDES Replicator is available as a docker image: `ghcr.io/osoc21/ldes-replicator`
 
 ### Requirements
 
@@ -73,15 +123,26 @@ A redis server and a backend server of your choice are needed for the LDES repli
 ```shell
 docker run \
  -e URLS=ldes.example \
- -e STATE_CONFIG={"host":"127.0.0.1", "id":"replicator"} \
+ -e 'STATE_CONFIG={"host":"127.0.0.1", "id":"replicator"}' \
  -e CONNECTORS=[0] \
  -e CONNECTOR_0_TYPE=ldes-backend-connector \
- -e CONNECTOR_0_CONFIG={...} \
+ -e 'CONNECTOR_0_CONFIG={...}' \
+ ghcr.io/osoc21/ldes-replicator:latest
+```
+
+or, if a config file is available:
+
+```shell
+docker run \
+ -v $(pwd)/config.json:./config.json
  ghcr.io/osoc21/ldes-replicator:latest
 ```
 
 ### Environment Variables
 
+Environment variables can be used to automatically create a [Config file](#config-file),
+otherwise those can be ignored and a config file mounted at `./config.json`.
+
 - `URLS`: comma-separated list of LDES URLs.
 - `STATE_CONFIG`: JSON object with the Redis connection settings.
 
@@ -102,48 +163,84 @@ docker run \
 - `CONNECTOR_<ID>_TYPE`: Name of the NPM package to use as a backend connector, a list of those can be found below.
 - `CONNECTOR_<ID>_CONFIG`: JSON object with the backend connector settings, those will depend on the connector used.
 
-### Connectors
+## Connectors
 
-#### @ldes/ldes-postgres-connector
+### Generic settings
+
+The connectors will try to implement generic settings to allow a more customizable replication:
+
+```js
+CONFIG:
+{
+ // Handles versioning, a SHACL shape is required. If not provided all the versions will be kept.
+ versions: {
+ // Amount of versions to keep
+ amount: number,
+ // Field of the LDES that identifies versions of the same element
+ identifier: string,
+ // Field of the LDES used to sort the versions (and get rid of the older ones)
+ sorter: string,
+ },
+ "other connector specific settings": ...
+}
+```
+
+### @ldes/ldes-postgres-connector
 
 Stores the Linked Data Events inside a postgres DB backend.
 
 ```js
 CONFIG:
 {
- //The name of the table the events will be stored in
- "tableName": string,
  //The Postgres server credentials
  "username": string,
  "password": string,
  "database": string,
  //The Postgres server hostname/IP address and port
  "hostname": string,
  "port": number
- //How many versions of an LDES element should be kept (0 => any amount)
- "amountOfVersions": number
 }
 ```
 
-#### @ldes/ldes-mongodb-connector
+### @ldes/ldes-mongodb-connector
 
 Stores the Linked Data Events inside a MongoDB backend.
 
 ```js
 CONFIG:
 {
- //The MongoDB server credentials
+ //The MongoDB server credentials, if needed
  "username": string,
  "password": string,
  "database": string,
  //The MongoDB server hostname/IP address and port
  "hostname": string,
  "port": number
- //How many versions of an LDES element should be kept (0 => any amount)
- "amountOfVersions": number
 }
 ```
 
+### @ldes/ldes-graphdb-connector and @ldes/ldes-graphdb-version-materialization-connector
+
+Stores the Linked Data Events inside a GraphDB backend.
+
+```js
+CONFIG:
+{
+ //The GraphDB server credentials, if needed
+ "password": string,
+ "database": string,
+ //The GraphDB server connection string and the repository to use
+ "baseUrl": string,
+ "repository": number,
+ //Each ldes is stored in a different graph, with this you can provide their URI prefix
+ "graphPrefix": string
+ // Field of the LDES to use as a label, this will create a Lucine connector for full-text search
+ "luceneLabel": string
+}
+```
+
+_Note: The version-materialization connector requires the versions setting (specifically the identifier field)._
+
 ### Docker-compose
 
 Docker compose can be used to host the Redis server, LDES Replicator and on the same dedicated server.
@@ -167,8 +264,6 @@ services:
  CONNECTORS: '[postgres]'
  CONNECTOR_postgres_TYPE: '@ldes/ldes-postgres-connector'
  CONNECTOR_postgres_CONFIG: '{
- "amountOfVersions":0,
- "tableName":"ldes",
  "username":"postgres",
  "password":"postgres",
  "database":"postgres",
@@ -212,8 +307,6 @@ releases:
  PG:
  type: '@ldes/ldes-postgres-connector'
  settings:
- amountOfVersions: 0
- tableName: 'ldes'
  username: 'postgres'
  password: 'postgres'
  database: 'postgres'

config.json.example

@@ -13,6 +13,7 @@
  "id": "replicator",
  "host": "localhost"
  },
+ "disable_pulling": true,
  "polling_interval": 5000
  },
  "connectors": {

packages/ldes-dummy-connector/lib/DummyConnector.ts

@@ -3,13 +3,6 @@ import type { IConfigConnector, IWritableConnector, LdesShape } from '@ldes/type
 export class DummyConnector implements IWritableConnector {
  private readonly members: any[];
 
- /**
- * Templates for the backend generator.
- */
- public static composeTemplate = ``;
-
- public static helmTemplate = ``;
-
  public constructor(config: IConfigConnector, shape: LdesShape, id: string) {
  this.members = [];
  console.log('Shape:');

packages/ldes-graphdb-connector/index.ts

@@ -0,0 +1 @@
+export * from './lib/GraphDbConnector';