Merge branch 'eXist-db:develop' into fix/3620

.github/workflows/ci-deploy.yml

@@ -24,8 +24,8 @@ jobs:
  uses: actions/cache@v3
  with:
  path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2
+ key: deploy-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+ restore-keys: deploy-${{ runner.os }}-maven
  - name: Install bats
  run: sudo apt-get install bats 
  - name: Build images
@@ -60,12 +60,12 @@ jobs:
  run: mvn -q -Ddocker.tag=release -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push
  working-directory: ./exist-docker
  # NOTE (DP): This is for debugging, publishes an experimental image from inside PRs against develop
- # - name: Publish experimental images
- #  if: github.base_ref == 'develop'
- #  env:
- #  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- #  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
- #  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
- # run: mvn -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push
- #  working-directory: ./exist-docker 
+ - name: Publish experimental images
+ if: github.base_ref == 'develop'
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+ DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+  run: mvn -q -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push
+ working-directory: ./exist-docker 
 

.github/workflows/ci-test.yml

@@ -30,38 +30,32 @@ jobs:
  cache: 'maven'
  - run: mvn -V -B dependency-check:check
  timeout-minutes: 60
- documentation:
- name: Javadoc
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-java@v3
- with:
- distribution: temurin
- java-version: ${{ env.DEV_JDK }}
- cache: 'maven'
- - run: mvn -V -B -q -T 2C install javadoc:javadoc -DskipTests -D'dependency-check.skip' -D'license.skip' --projects '!exist-distribution,!exist-installer' --also-make
  test:
- name: (JDK ${{ matrix.jdk }} / ${{ matrix.os }}) Test
+ name: ${{ matrix.os }} Test
  strategy:
  fail-fast: false
  matrix:
  os: [ubuntu-latest, windows-latest, macOS-latest]
- jdk: [ '17' ]
  runs-on: ${{ matrix.os }}
  steps:
  - uses: actions/checkout@v3
- - name: Set up JDK ${{ matrix.jdk }}
+ - name: Set up JDK
  uses: actions/setup-java@v3
  with:
  distribution: temurin
- java-version: ${{ matrix.jdk }}
+ java-version: ${{ env.DEV_JDK }}
  cache: 'maven'
+ - name: Maven Build
+ timeout-minutes: 10
+ run: mvn -V -B -T 1C compile test-compile -DtrimStackTrace=false -D'dependency-check.skip' -D'license.skip'
  - name: Maven Test
  timeout-minutes: 60
- run: mvn -V -B verify -D'dependency-check.skip' -D'license.skip'
- - name: Maven Code Coverage
- if: ${{ github.ref == 'refs/heads/develop' && matrix.jdk == env.DEV_JDK && matrix.os == 'ubuntu-latest' }}
+ run: mvn -V -B verify -DtrimStackTrace=false -D'dependency-check.skip' -D'license.skip'
+ - name: Javadoc (Linux only)
+ if: ${{ matrix.os == 'ubuntu-latest' }}
+ run: mvn -V -B -q -T 1C install javadoc:javadoc -DskipTests -D'dependency-check.skip' -D'license.skip' --projects '!exist-distribution,!exist-installer' --also-make
+ - name: Maven Code Coverage (Develop branch on Linux only)
+ if: ${{ github.ref == 'refs/heads/develop' && matrix.os == 'ubuntu-latest' }}
  env:
  CI_NAME: github
  BRANCH_NAME_OR_REF: ${{ github.head_ref || github.ref }}
@@ -73,7 +67,7 @@ jobs:
  if: always()
  uses: actions/upload-artifact@v3
  with:
- name: ${{ runner.os }}-jdk${{ matrix.jdk }}-build-logs
+ name: ${{ runner.os }}-build-logs
  retention-days: 5
  path: |
  **/*.jfr

.github/workflows/ci-xqts.yml

@@ -14,7 +14,12 @@ jobs:
  with:
  distribution: temurin
  java-version: '17'
- cache: 'maven'
+ - name: Cache Maven packages
+ uses: actions/cache@v3
+ with:
+ path: ~/.m2
+ key: xqts-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+ restore-keys: xqts-${{ runner.os }}-maven
  - name: Maven XQTS Build
  run: mvn -V -B clean package -DskipTests -Ddependency-check.skip=true --projects exist-xqts --also-make
  - name: Run XQTS

.github/workflows/sonarcloud.yml

@@ -23,14 +23,14 @@ jobs:
  uses: actions/cache@v3
  with:
  path: ~/.sonar/cache
- key: ${{ runner.os }}-sonar
- restore-keys: ${{ runner.os }}-sonar
+ key: sonarcloud-${{ runner.os }}-cache-${{ hashFiles('**/pom.xml') }}
+ restore-keys: sonarcloud-${{ runner.os }}-cache
  - name: Cache Maven packages
  uses: actions/cache@v3
  with:
  path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2
+ key: sonarcloud-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+ restore-keys: sonarcloud-${{ runner.os }}-maven
  - name: Analyze
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any

exist-core/pom.xml

@@ -162,7 +162,7 @@
  <dependency>
  <groupId>com.fasterxml.uuid</groupId>
  <artifactId>java-uuid-generator</artifactId>
- <version>4.1.0</version>
+ <version>4.2.0</version>
  </dependency>
 
  <dependency>
@@ -212,8 +212,8 @@
  </dependency>
 
  <dependency>
- <groupId>cglib</groupId>
- <artifactId>cglib</artifactId>
+ <groupId>net.bytebuddy</groupId>
+ <artifactId>byte-buddy</artifactId>
  </dependency>
 
  <dependency>

exist-core/src/main/java/org/exist/dom/persistent/NodeProxy.java

@@ -681,6 +681,8 @@ public static int nodeType2XQuery(final short nodeType) {
  return Type.ATTRIBUTE;
  case Node.TEXT_NODE:
  return Type.TEXT;
+ case Node.CDATA_SECTION_NODE:
+ return Type.CDATA_SECTION;
  case Node.PROCESSING_INSTRUCTION_NODE:
  return Type.PROCESSING_INSTRUCTION;
  case Node.COMMENT_NODE:

exist-core/src/main/java/org/exist/launcher/ConfigurationDialog.java

@@ -97,6 +97,12 @@ public ConfigurationDialog(Consumer<Boolean> callback) {
  if (ports.containsKey("jetty.port")) {
  httpPort.setValue(ports.get("jetty.port"));
  }
+ if (ports.containsKey("jetty.http.port")) {
+ httpPort.setValue(ports.get("jetty.http.port"));
+ }
+ if (ports.containsKey("ssl.port")) {
+ sslPort.setValue(ports.get("ssl.port"));
+ }
  if (ports.containsKey("jetty.ssl.port")) {
  sslPort.setValue(ports.get("jetty.ssl.port"));
  }

exist-core/src/main/java/org/exist/launcher/ConfigurationUtility.java

@@ -93,7 +93,7 @@ private static void getJettyPorts(Map<String, Integer> ports, Path jettyConfig)
  final int status = reader.next();
  if (status == XMLStreamReader.START_ELEMENT && "SystemProperty".equals(reader.getLocalName())) {
  final String name = reader.getAttributeValue(null, "name");
- if (name != null && (name.equals("jetty.port") || name.equals("jetty.ssl.port"))) {
+ if (name != null && (name.equals("jetty.http.port") || name.equals("jetty.ssl.port"))) {
  final String defaultValue = reader.getAttributeValue(null, "default");
  if (defaultValue != null) {
  try {

exist-core/src/main/java/org/exist/repo/ClasspathHelper.java

@@ -39,6 +39,7 @@
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Collection;
 import java.util.Optional;
 import java.util.Set;
@@ -74,7 +75,7 @@ public static void updateClasspath(BrokerPool pool, org.expath.pkg.repo.Package
  final Path packageDir = resolver.resolveResourceAsFile(".");
  final Classpath cp = new Classpath();
  try {
- scanPackageDir(cp, packageDir);
+ scanPackageDir(pkg, cp, packageDir);
  ((EXistClassLoader)loader).addURLs(cp);
  } catch (final IOException e) {
  LOG.warn("An error occurred while updating classpath for package {}", pkg.getName(), e);
@@ -93,7 +94,7 @@ private static void scanPackages(BrokerPool pool, Classpath classpath) {
  try {
  final FileSystemStorage.FileSystemResolver resolver = (FileSystemStorage.FileSystemResolver) pkg.getResolver();
  final Path packageDir = resolver.resolveResourceAsFile(".");
- scanPackageDir(classpath, packageDir);
+ scanPackageDir(pkg, classpath, packageDir);
  } catch (final IOException e) {
  LOG.warn("An error occurred while updating classpath for package {}", pkg.getName(), e);
  }
@@ -157,7 +158,7 @@ private static boolean isCompatible(final Package pkg) throws PackageException {
  }
  }
 
- private static void scanPackageDir(Classpath classpath, Path module) throws IOException {
+ private static void scanPackageDir(Package pkg, Classpath classpath, Path module) throws IOException {
  final Path dotExist = module.resolve(".exist");
  if (Files.exists(dotExist)) {
  if (!Files.isDirectory(dotExist)) {
@@ -169,7 +170,18 @@ private static void scanPackageDir(Classpath classpath, Path module) throws IOEx
  try (final BufferedReader reader = Files.newBufferedReader(cp)) {
  String line;
  while ((line = reader.readLine()) != null) {
- classpath.addComponent(line);
+ Path p = Paths.get(line);
+ if (!p.isAbsolute()) {
+ final FileSystemStorage.FileSystemResolver res = (FileSystemStorage.FileSystemResolver) pkg.getResolver();
+ p = res.resolveComponentAsFile(line);
+ }
+ p = p.normalize().toAbsolutePath();
+
+ if (Files.exists(p)) {
+ classpath.addComponent(p.toString());
+ } else {
+ LOG.warn("Unable to add '" + p + "' to the classpath for EXPath package: " + pkg.getName() + ", as the file does not exist!");
+ }
  }
  }
  }

exist-core/src/main/java/org/exist/repo/ExistPkgExtension.java

@@ -32,15 +32,12 @@
 import javax.xml.stream.XMLStreamConstants;
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamReader;
-import javax.xml.transform.stream.StreamSource;
 import java.io.IOException;
 import java.io.Writer;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Set;
 
 /**
  * Handle the exist.xml descriptor in an EXPath package.
@@ -158,23 +155,22 @@ private void setupPackage(Package pkg, ExistPkgInfo info)
  throw new PackageException("Impossible to create directory: " + exist);
  }
  }
- final Set<String> jars = info.getJars();
+
  try(final Writer out = Files.newBufferedWriter(classpath)) {
- for (final String jar : jars) {
- StreamSource jar_src;
+ for (final String jar : info.getJars()) {
+
  try {
- jar_src = res.resolveComponent(jar);
+ res.resolveComponent(jar);
  } catch (final NotExistException ex) {
- final String msg = "Inconsistent package descriptor, the JAR file is not in the package: ";
+ final String msg = "Inconsistent package descriptor, the JAR file is not in the EXPath package: ";
  throw new PackageException(msg + jar, ex);
  }
- final URI uri = URI.create(jar_src.getSystemId());
- final Path file = Paths.get(uri);
- out.write(file.normalize().toString());
- out.write("\n");
+
+ out.write(jar);
+ out.write('\n');
  }
  } catch (final IOException ex) {
- throw new PackageException("Error writing the eXist classpath file: " + classpath, ex);
+ throw new PackageException("Error writing the eXist classpath file '" + classpath + "' for the EXPath package: " + pkg.getName(), ex);
  }
  }
 

exist-core/src/main/java/org/exist/storage/BrokerPool.java

@@ -50,8 +50,8 @@
 import org.exist.scheduler.Scheduler;
 import org.exist.scheduler.impl.QuartzSchedulerImpl;
 import org.exist.scheduler.impl.SystemTaskJobImpl;
-import org.exist.security.*;
 import org.exist.security.SecurityManager;
+import org.exist.security.*;
 import org.exist.security.internal.SecurityManagerImpl;
 import org.exist.storage.blob.BlobStore;
 import org.exist.storage.blob.BlobStoreImplService;
@@ -116,6 +116,7 @@ public class BrokerPool extends BrokerPools implements BrokerPoolConstants, Data
 
  private final XQuery xqueryService = new XQuery();
 
+ private AtomicLazyVal<SaxonConfiguration> saxonConfiguration = new AtomicLazyVal<>(() -> SaxonConfiguration.loadConfiguration(this));
 
  //TODO : make it non-static since every database instance may have its own policy.
  //TODO : make a default value that could be overwritten by the configuration
@@ -374,13 +375,6 @@ public String getStatus() {
 
  private StartupTriggersManager startupTriggersManager;
 
- /**
- * Configuration for Saxon.
- *
- * One instance per-database, lazily initialised.
- */
- private AtomicLazyVal<net.sf.saxon.Configuration> saxonConfig = new AtomicLazyVal<>(net.sf.saxon.Configuration::newConfiguration);
-
  /**
  * Creates and configures the database instance.
  *
@@ -1926,7 +1920,11 @@ public void registerCollectionTrigger(final Class<? extends CollectionTrigger> c
  }
 
  public net.sf.saxon.Configuration getSaxonConfiguration() {
- return saxonConfig.get();
+ return saxonConfiguration.get().getConfiguration();
+ }
+
+ public net.sf.saxon.s9api.Processor getSaxonProcessor() {
+ return saxonConfiguration.get().getProcessor();
  }
 
  /**

exist-core/src/main/java/org/exist/util/Configuration.java

@@ -260,6 +260,12 @@ public Configuration(String configFilename, Optional<Path> existHomeDirname) thr
  configureTransformer((Element)transformers.item(0));
  }
 
+ //saxon settings (most importantly license file for PE or EE features)
+ final NodeList saxon = doc.getElementsByTagName(SaxonConfiguration.SAXON_CONFIGURATION_ELEMENT_NAME);
+ if( saxon.getLength() > 0 ) {
+ configureSaxon((Element)saxon.item(0));
+ }
+
  //parser settings
  final NodeList parsers = doc.getElementsByTagName(HtmlToXmlParser.PARSER_ELEMENT_NAME);
  if(parsers.getLength() > 0) {
@@ -538,6 +544,14 @@ private void configureXUpdate( Element xupdate ) throws NumberFormatException
  }
  }
 
+ private void configureSaxon( Element saxon )
+ {
+ final String configurationFile = getConfigAttributeValue( saxon,
+ SaxonConfiguration.SAXON_CONFIGURATION_FILE_ATTRIBUTE);
+ if (configurationFile != null) {
+ config.put(SaxonConfiguration.SAXON_CONFIGURATION_FILE_PROPERTY, configurationFile);
+ }
+ }
 
  private void configureTransformer( Element transformer )
  {