From bc2c366c83d428fce74114e12f55c827f3f544bd Mon Sep 17 00:00:00 2001 From: psilberk Date: Tue, 22 Apr 2025 09:31:27 -0700 Subject: [PATCH 01/64] Adding Oracle Vector Store --- .../OracleVectorStoreQueryProvider.java | 91 ++++++++ .../memory/VectorStoreWithOracle.java | 210 ++++++++++++++++++ 2 files changed, 301 insertions(+) create mode 100644 data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java new file mode 100644 index 00000000..f179d0be --- /dev/null +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -0,0 +1,91 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +import javax.annotation.Nonnull; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; + +public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { + + // This could be removed if super.collectionTable made protected + private final String collectionsTable; + + private OracleVectorStoreQueryProvider(@Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables) { + super(dataSource, collectionsTable, prefixForCollectionTables); + this.collectionsTable = collectionsTable; + } + + @Override + public void prepareVectorStore() { + String createCollectionsTable = formatQuery( + "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY)", + validateSQLidentifier(collectionsTable)); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement createTable = connection.prepareStatement(createCollectionsTable)) { + createTable.execute(); + } catch (SQLException e) { + throw new SKException("Failed to prepare vector store", e); + } + } + + @Override + public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + + // Using hsqldb impl + + + super.upsertRecords(collectionName, records, recordDefinition, options); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder + extends JDBCVectorStoreQueryProvider.Builder { + + private DataSource dataSource; + private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; + return this; + } + + /** + * Sets the collections table name. + * @param collectionsTable the collections table name + * @return the builder + */ + public Builder withCollectionsTable(String collectionsTable) { + this.collectionsTable = validateSQLidentifier(collectionsTable); + return this; + } + + /** + * Sets the prefix for collection tables. + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); + return this; + } + + @Override + public OracleVectorStoreQueryProvider build() { + return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, prefixForCollectionTables); + } + } +} \ No newline at end of file diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java new file mode 100644 index 00000000..9d142507 --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java @@ -0,0 +1,210 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.syntaxexamples.memory; + +import com.azure.ai.openai.OpenAIAsyncClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; +import com.azure.core.credential.KeyCredential; +import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; + +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.postgresql.ds.PGSimpleDataSource; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public class VectorStoreWithOracle { + + static class GitHubFile { + @VectorStoreRecordKey + private final String id; + @VectorStoreRecordData + private final String description; + @VectorStoreRecordData + private final String link; + @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE) + private final List embedding; + + public GitHubFile() { + this(null, null, null, Collections.emptyList()); + } + + public GitHubFile( + String id, + String description, + String link, + List embedding) { + this.id = id; + this.description = description; + this.link = link; + this.embedding = embedding; + } + + public String getId() { + return id; + } + + public String getDescription() { + return description; + } + + public String getLink() { + return link; + } + + public List getEmbedding() { + return embedding; + } + + static String encodeId(String realId) { + byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8)); + return new String(bytes, StandardCharsets.UTF_8); + } + } + + // Run a PostgreSQL server with: + // docker run -d --name pgvector-container -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=root -e POSTGRES_DB=sk -p 5432:5432 pgvector/pgvector:pg17 + + public static void main(String[] args) throws SQLException { + System.out.println("=============================================================="); + System.out.println("============== Oracle Vector Store Example ==================="); + System.out.println("=============================================================="); + + OpenAIAsyncClient client; + + if (AZURE_CLIENT_KEY != null) { + client = new OpenAIClientBuilder() + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); + + } else { + client = new OpenAIClientBuilder() + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); + } + + // Create an OpenAI text embedding generation service + var embeddingGeneration = OpenAITextEmbeddingGenerationService.builder() + .withOpenAIAsyncClient(client) + .withModelId(MODEL_ID) + .withDimensions(EMBEDDING_DIMENSIONS) + .build(); + + storeAndSearch(embeddingGeneration); + } + + public static void storeAndSearch(OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Configure the data source + PGSimpleDataSource dataSource = new PGSimpleDataSource(); + dataSource.setUrl("jdbc:postgresql://localhost:5432/sk"); + dataSource.setUser("postgres"); + dataSource.setPassword("root"); + + // Build a query provider + // Other available query providers are PostgreSQLVectorStoreQueryProvider and SQLiteVectorStoreQueryProvider + var queryProvider = PostgreSQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + + // Build a vector store + var jdbcVectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + // Set up the record collection to use + String collectionName = "skgithubfiles"; + var collection = jdbcVectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(GitHubFile.class) + .build()); + + // Create collection if it does not exist and store data + collection + .createCollectionIfNotExistsAsync() + .then(storeData(collection, embeddingGeneration, sampleData())) + .block(); + + // Search for results + var results = search("How to get started", collection, embeddingGeneration).block(); + + if (results == null || results.getTotalCount() == 0) { + System.out.println("No search results found."); + return; + } + var searchResult = results.getResults().get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); + } + + private static Mono> search( + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Generate embeddings for the search text and search for the closest records + return embeddingGeneration.generateEmbeddingAsync(searchText) + .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); + } + + private static Mono> storeData( + VectorStoreRecordCollection recordStore, + OpenAITextEmbeddingGenerationService embeddingGeneration, + Map data) { + + return Flux.fromIterable(data.entrySet()) + .flatMap(entry -> { + System.out.println("Save '" + entry.getKey() + "' to memory."); + + // Generate embeddings for the data and store it + return embeddingGeneration + .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) + .flatMap(embeddings -> { + GitHubFile gitHubFile = new GitHubFile( + GitHubFile.encodeId(entry.getKey()), + entry.getValue(), + entry.getKey(), + embeddings.get(0).getVector()); + return recordStore.upsertAsync(gitHubFile, null); + }); + }) + .collectList(); + } + + private static Map sampleData() { + return Arrays.stream(new String[][] { + { "https://github.com/microsoft/semantic-kernel/blob/main/README.md", + "README: Installation, getting started with Semantic Kernel, and how to contribute" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/samples/notebooks/dotnet/02-running-prompts-from-file.ipynb", + "Jupyter notebook describing how to pass prompts from a file to a semantic skill or function" }, + { "https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT", + "Sample demonstrating how to create a chat skill interfacing with ChatGPT" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs", + "C# class that defines a volatile embedding store" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/samples/dotnet/KernelHttpServer/README.md", + "README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/samples/apps/chat-summary-webapp-react/README.md", + "README: README associated with a sample chat summary react-based webapp" }, + }).collect(Collectors.toMap(element -> element[0], element -> element[1])); + } +} From ba1aa96d690d59933fa0f5b86557de221a73c69a Mon Sep 17 00:00:00 2001 From: psilberk Date: Fri, 25 Apr 2025 16:20:54 -0700 Subject: [PATCH 02/64] Upsert and Main sample --- data/semantickernel-data-jdbc/pom.xml | 5 + .../OracleVectorStoreQueryProvider.java | 61 +++++- pom.xml | 1 + .../semantickernel-syntax-examples/pom.xml | 11 + .../memory/VectorStoreWithOracle.java | 200 +++--------------- .../data/vectorstores/oracle/Main.java | 72 +++++++ 6 files changed, 171 insertions(+), 179 deletions(-) create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 624779e9..1f2e1a28 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -63,5 +63,10 @@ sqlite-jdbc 3.47.0.0 + + com.oracle.database.jdbc + ojdbc11 + 23.7.0.25.01 + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index f179d0be..10af511c 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -1,6 +1,9 @@ package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; @@ -18,9 +21,14 @@ public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider // This could be removed if super.collectionTable made protected private final String collectionsTable; - private OracleVectorStoreQueryProvider(@Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables) { + // This could be common to all query providers + private final ObjectMapper objectMapper; + + private OracleVectorStoreQueryProvider(@Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables, + ObjectMapper objectMapper) { super(dataSource, collectionsTable, prefixForCollectionTables); this.collectionsTable = collectionsTable; + this.objectMapper = objectMapper; } @Override @@ -37,13 +45,50 @@ public void prepareVectorStore() { } } + @Override + public void createCollection(String collectionName, + VectorStoreRecordDefinition recordDefinition) { + // TODO Override implementation. Eg: mapping TEXT to VARCHAR + super.createCollection(collectionName, recordDefinition); + } + @Override public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { - // Using hsqldb impl + // TODO look for public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { + + // TODO Make this a MERGE query + +// String upsertStatemente = formatQuery(""" +// MERGE INTO %s EXIST_REC USING (SELECT ? AS ID) NEW_REC ON (EXIST_REC.%s = NEW_REC.ID) +// WHEN MATACHED THEN UPDATE SET EXISTING REC +// """, +// getCollectionTableName(collectionName), +// recordDefinition.getKeyField().getName(), +// getQueryColumnsFromFields(fields), +// getWildcardString(fields.size()), +// onDuplicateKeyUpdate);super.upsertRecords(collectionName, records, recordDefinition, options); + String query = formatQuery("INSERT INTO %s (%s, %s, %s) values (?, ?, ?)", + getCollectionTableName(collectionName), + recordDefinition.getAllFields().get(0).getStorageName(), + recordDefinition.getAllFields().get(1).getStorageName(), + recordDefinition.getAllFields().get(2).getStorageName()); - super.upsertRecords(collectionName, records, recordDefinition, options); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { + for (Object record : records) { + JsonNode jsonNode = objectMapper.valueToTree(record); + for (int i = 0; i < 3; i++) { + statement.setObject(i + 1, jsonNode + .get(recordDefinition.getAllFields().get(i).getStorageName()).asText()); + } + statement.addBatch(); + } + statement.executeBatch(); + } catch (SQLException e) { + throw new SKException("Failed to upsert records", e); + } } public static Builder builder() { @@ -56,6 +101,7 @@ public static class Builder private DataSource dataSource; private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + private ObjectMapper objectMapper = new ObjectMapper(); @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withDataSource(DataSource dataSource) { @@ -83,9 +129,16 @@ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { return this; } + public Builder withObjectMapper( + ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + @Override public OracleVectorStoreQueryProvider build() { - return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, prefixForCollectionTables); + return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, + prefixForCollectionTables, objectMapper); } } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 30963f54..e12980a2 100644 --- a/pom.xml +++ b/pom.xml @@ -71,6 +71,7 @@ semantickernel-bom semantickernel-api semantickernel-experimental + samples aiservices/openai aiservices/google aiservices/huggingface diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 1f6bc181..1abf1174 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -131,6 +131,17 @@ com.github.victools jsonschema-module-jackson + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + 1.4.4-SNAPSHOT + + + com.microsoft.semantic-kernel + semantickernel-learn-resources + 1.4.4-SNAPSHOT + compile + diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java index 9d142507..3d037c70 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java @@ -1,210 +1,60 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.samples.syntaxexamples.memory; -import com.azure.ai.openai.OpenAIAsyncClient; -import com.azure.ai.openai.OpenAIClientBuilder; -import com.azure.core.credential.AzureKeyCredential; -import com.azure.core.credential.KeyCredential; -import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollection; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; -import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; - -import java.nio.charset.StandardCharsets; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; import java.sql.SQLException; -import java.util.Arrays; -import java.util.Base64; import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.postgresql.ds.PGSimpleDataSource; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; +import oracle.jdbc.datasource.impl.OracleDataSource; public class VectorStoreWithOracle { - static class GitHubFile { - @VectorStoreRecordKey - private final String id; - @VectorStoreRecordData - private final String description; - @VectorStoreRecordData - private final String link; - @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE) - private final List embedding; - - public GitHubFile() { - this(null, null, null, Collections.emptyList()); - } - - public GitHubFile( - String id, - String description, - String link, - List embedding) { - this.id = id; - this.description = description; - this.link = link; - this.embedding = embedding; - } - - public String getId() { - return id; - } - - public String getDescription() { - return description; - } - - public String getLink() { - return link; - } - - public List getEmbedding() { - return embedding; - } - - static String encodeId(String realId) { - byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8)); - return new String(bytes, StandardCharsets.UTF_8); - } - } - - // Run a PostgreSQL server with: - // docker run -d --name pgvector-container -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=root -e POSTGRES_DB=sk -p 5432:5432 pgvector/pgvector:pg17 - public static void main(String[] args) throws SQLException { System.out.println("=============================================================="); System.out.println("============== Oracle Vector Store Example ==================="); System.out.println("=============================================================="); - OpenAIAsyncClient client; - - if (AZURE_CLIENT_KEY != null) { - client = new OpenAIClientBuilder() - .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) - .endpoint(CLIENT_ENDPOINT) - .buildAsyncClient(); - - } else { - client = new OpenAIClientBuilder() - .credential(new KeyCredential(CLIENT_KEY)) - .buildAsyncClient(); - } - - // Create an OpenAI text embedding generation service - var embeddingGeneration = OpenAITextEmbeddingGenerationService.builder() - .withOpenAIAsyncClient(client) - .withModelId(MODEL_ID) - .withDimensions(EMBEDDING_DIMENSIONS) - .build(); - - storeAndSearch(embeddingGeneration); - } - - public static void storeAndSearch(OpenAITextEmbeddingGenerationService embeddingGeneration) { // Configure the data source - PGSimpleDataSource dataSource = new PGSimpleDataSource(); - dataSource.setUrl("jdbc:postgresql://localhost:5432/sk"); - dataSource.setUser("postgres"); - dataSource.setPassword("root"); + OracleDataSource dataSource = new OracleDataSource(); + dataSource.setURL("jdbc:oracle:thin:@localhost:1521/FREEPDB1"); + dataSource.setUser("scott"); + dataSource.setPassword("tiger"); // Build a query provider - // Other available query providers are PostgreSQLVectorStoreQueryProvider and SQLiteVectorStoreQueryProvider - var queryProvider = PostgreSQLVectorStoreQueryProvider.builder() + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() .withDataSource(dataSource) .build(); // Build a vector store - var jdbcVectorStore = JDBCVectorStore.builder() + JDBCVectorStore vectorStore = JDBCVectorStore.builder() .withDataSource(dataSource) .withOptions(JDBCVectorStoreOptions.builder() .withQueryProvider(queryProvider) .build()) .build(); - // Set up the record collection to use - String collectionName = "skgithubfiles"; - var collection = jdbcVectorStore.getCollection(collectionName, - JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(GitHubFile.class) - .build()); - - // Create collection if it does not exist and store data - collection - .createCollectionIfNotExistsAsync() - .then(storeData(collection, embeddingGeneration, sampleData())) + // Get a collection from the vector store + VectorStoreRecordCollection collection = + vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + // Create the collection if it doesn't exist yet. + collection.createCollectionAsync().block(); + + collection.upsertAsync(new Hotel("1", + "HotelOne", + "Desc for HotelOne", + Collections.emptyList(), Collections.emptyList()), + null) .block(); - // Search for results - var results = search("How to get started", collection, embeddingGeneration).block(); - - if (results == null || results.getTotalCount() == 0) { - System.out.println("No search results found."); - return; - } - var searchResult = results.getResults().get(0); - System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", - searchResult.getScore(), searchResult.getRecord().link, - searchResult.getRecord().description); } - private static Mono> search( - String searchText, - VectorStoreRecordCollection recordCollection, - OpenAITextEmbeddingGenerationService embeddingGeneration) { - // Generate embeddings for the search text and search for the closest records - return embeddingGeneration.generateEmbeddingAsync(searchText) - .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); - } - - private static Mono> storeData( - VectorStoreRecordCollection recordStore, - OpenAITextEmbeddingGenerationService embeddingGeneration, - Map data) { - - return Flux.fromIterable(data.entrySet()) - .flatMap(entry -> { - System.out.println("Save '" + entry.getKey() + "' to memory."); - - // Generate embeddings for the data and store it - return embeddingGeneration - .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) - .flatMap(embeddings -> { - GitHubFile gitHubFile = new GitHubFile( - GitHubFile.encodeId(entry.getKey()), - entry.getValue(), - entry.getKey(), - embeddings.get(0).getVector()); - return recordStore.upsertAsync(gitHubFile, null); - }); - }) - .collectList(); - } - - private static Map sampleData() { - return Arrays.stream(new String[][] { - { "https://github.com/microsoft/semantic-kernel/blob/main/README.md", - "README: Installation, getting started with Semantic Kernel, and how to contribute" }, - { "https://github.com/microsoft/semantic-kernel/blob/main/samples/notebooks/dotnet/02-running-prompts-from-file.ipynb", - "Jupyter notebook describing how to pass prompts from a file to a semantic skill or function" }, - { "https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT", - "Sample demonstrating how to create a chat skill interfacing with ChatGPT" }, - { "https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs", - "C# class that defines a volatile embedding store" }, - { "https://github.com/microsoft/semantic-kernel/blob/main/samples/dotnet/KernelHttpServer/README.md", - "README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4" }, - { "https://github.com/microsoft/semantic-kernel/blob/main/samples/apps/chat-summary-webapp-react/README.md", - "README: README associated with a sample chat summary react-based webapp" }, - }).collect(Collectors.toMap(element -> element[0], element -> element[1])); - } } diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java new file mode 100644 index 00000000..7a7b2b8f --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; +import oracle.jdbc.datasource.impl.OracleDataSource; +import java.sql.SQLException; +import java.util.Collections; + +public class Main { + public static void main(String[] args) throws SQLException { + + // Configure the data source + OracleDataSource dataSource = new OracleDataSource(); + dataSource.setURL("jdbc:oracle:thin:@localhost:1521/FREEPDB1"); + dataSource.setUser("scott"); + dataSource.setPassword("tiger"); + + // Build a query provider + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + + // Build a vector store + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + // Get a collection from the vector store + VectorStoreRecordCollection collection = + vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + // Create the collection if it doesn't exist yet. + // TODO Override implementation to map TEXT to VARCHAR + // Craeted manually for the moment + //collection.createCollectionAsync().block(); + + // Upsert a record. + collection.upsertAsync(new Hotel("1", + "HotelOne", + "My Description for HotelOne", + Collections.emptyList(), Collections.emptyList()), + null) + .block(); + + // Retrieve the upserted record. +// var retrievedHotel = collection.getAsync("1", null).block(); + + // Generate a vector for your search text, using your chosen embedding generation implementation. + // Just showing a placeholder method here for brevity. +// var searchVector = generateEmbeddingsAsync( +// "I'm looking for a hotel where customer happiness is the priority.").block(); + + // Do the search. +// var searchResult = collection.searchAsync(searchVector, VectorSearchOptions.builder() +// .withTop(1).build()).block(); + +// Hotel record = searchResult.getResults().get(0).getRecord(); +// System.out.printf("Found hotel description: %s\n", record.getDescription()); + + } +} From 7d476fcaf0fe0505de5534e4f48ef667303bb061 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Thu, 22 May 2025 12:09:40 +0200 Subject: [PATCH 03/64] Split out the data into its own module --- data/semantickernel-data-hsqldb/pom.xml | 64 +++++++++++++++++++ .../HSQLDBVectorStoreQueryProvider.java | 0 data/semantickernel-data-jdbc/pom.xml | 2 +- .../jdbc/JDBCVectorStoreQueryProvider.java | 13 +++- .../jdbc/JDBCVectorStoreRecordCollection.java | 25 +------- .../jdbc/SQLVectorStoreQueryProvider.java | 12 ++++ data/semantickernel-data-mysql/pom.xml | 64 +++++++++++++++++++ .../mysql/MySQLVectorStoreQueryProvider.java | 0 data/semantickernel-data-postgres/pom.xml | 64 +++++++++++++++++++ .../PostgreSQLVectorDistanceFunction.java | 0 .../postgres/PostgreSQLVectorIndexKind.java | 0 .../PostgreSQLVectorStoreQueryProvider.java | 11 +++- .../PostgreSQLVectorStoreRecordMapper.java | 0 data/semantickernel-data-sqlite/pom.xml | 63 ++++++++++++++++++ .../SQLiteVectorStoreQueryProvider.java | 0 pom.xml | 34 ++++++++++ semantickernel-api-builders/pom.xml | 30 +++++++++ .../builders/SemanticKernelBuilder.java | 0 semantickernel-api-data/pom.xml | 60 +++++++++++++++++ .../data/VectorStoreTextSearch.java | 0 .../data/VectorStoreTextSearchOptions.java | 0 .../data/VolatileVectorStore.java | 0 ...ileVectorStoreCollectionSearchMapping.java | 0 .../VolatileVectorStoreRecordCollection.java | 0 ...ileVectorStoreRecordCollectionOptions.java | 0 .../filter/AnyTagEqualToFilterClause.java | 0 .../data/filter/EqualToFilterClause.java | 0 .../data/filter/FilterClause.java | 0 .../data/filter/FilterMapping.java | 0 .../DefaultTextSearchResultMapper.java | 0 .../DefaultTextSearchStringMapper.java | 0 .../data/textsearch/KernelSearchResults.java | 0 .../data/textsearch/TextSearch.java | 0 .../data/textsearch/TextSearchFilter.java | 0 .../data/textsearch/TextSearchOptions.java | 0 .../data/textsearch/TextSearchResult.java | 0 .../data/textsearch/TextSearchResultLink.java | 0 .../textsearch/TextSearchResultMapper.java | 0 .../data/textsearch/TextSearchResultName.java | 0 .../textsearch/TextSearchResultValue.java | 0 .../textsearch/TextSearchStringMapper.java | 0 .../data/vectorsearch/VectorOperations.java | 0 .../data/vectorsearch/VectorSearchFilter.java | 0 .../data/vectorsearch/VectorSearchResult.java | 0 .../vectorsearch/VectorSearchResults.java | 0 .../vectorsearch/VectorizableTextSearch.java | 0 .../data/vectorsearch/VectorizedSearch.java | 0 .../data/vectorstorage/VectorStore.java | 0 .../VectorStoreRecordCollection.java | 0 .../VectorStoreRecordCollectionOptions.java | 0 .../VectorStoreRecordMapper.java | 0 .../annotations/VectorStoreRecordData.java | 0 .../annotations/VectorStoreRecordKey.java | 0 .../annotations/VectorStoreRecordVector.java | 0 .../definition/DistanceFunction.java | 0 .../vectorstorage/definition/IndexKind.java | 0 .../VectorStoreRecordDataField.java | 0 .../VectorStoreRecordDefinition.java | 0 .../definition/VectorStoreRecordField.java | 0 .../definition/VectorStoreRecordKeyField.java | 0 .../VectorStoreRecordVectorField.java | 0 .../options/DeleteRecordOptions.java | 0 .../options/GetRecordOptions.java | 0 .../options/UpsertRecordOptions.java | 0 .../options/VectorSearchOptions.java | 0 semantickernel-api-exceptions/pom.xml | 43 +++++++++++++ .../exceptions/AIException.java | 0 .../exceptions/ConfigurationException.java | 0 .../exceptions/SKCheckedException.java | 0 .../exceptions/SKException.java | 0 semantickernel-api-localization/pom.xml | 31 +++++++++ .../localization/SemanticKernelResources.java | 0 .../localization/ResourceBundle.properties | 0 .../pom.xml | 49 ++++++++++++++ .../semantickernel/services/AIService.java | 0 .../services/textembedding/Embedding.java | 0 .../EmbeddingGenerationService.java | 0 .../TextEmbeddingGenerationService.java | 0 semantickernel-api/pom.xml | 20 ++++++ 79 files changed, 560 insertions(+), 25 deletions(-) create mode 100644 data/semantickernel-data-hsqldb/pom.xml rename data/{semantickernel-data-jdbc => semantickernel-data-hsqldb}/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java (100%) create mode 100644 data/semantickernel-data-mysql/pom.xml rename data/{semantickernel-data-jdbc => semantickernel-data-mysql}/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java (100%) create mode 100644 data/semantickernel-data-postgres/pom.xml rename data/{semantickernel-data-jdbc => semantickernel-data-postgres}/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java (100%) rename data/{semantickernel-data-jdbc => semantickernel-data-postgres}/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java (100%) rename data/{semantickernel-data-jdbc => semantickernel-data-postgres}/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java (97%) rename data/{semantickernel-data-jdbc => semantickernel-data-postgres}/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java (100%) create mode 100644 data/semantickernel-data-sqlite/pom.xml rename data/{semantickernel-data-jdbc => semantickernel-data-sqlite}/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java (100%) create mode 100644 semantickernel-api-builders/pom.xml rename {semantickernel-api => semantickernel-api-builders}/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java (100%) create mode 100644 semantickernel-api-data/pom.xml rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java (100%) create mode 100644 semantickernel-api-exceptions/pom.xml rename {semantickernel-api => semantickernel-api-exceptions}/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java (100%) rename {semantickernel-api => semantickernel-api-exceptions}/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java (100%) rename {semantickernel-api => semantickernel-api-exceptions}/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java (100%) rename {semantickernel-api => semantickernel-api-exceptions}/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java (100%) create mode 100644 semantickernel-api-localization/pom.xml rename {semantickernel-api => semantickernel-api-localization}/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java (100%) rename {semantickernel-api => semantickernel-api-localization}/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties (100%) create mode 100644 semantickernel-api-textembedding-services/pom.xml rename {semantickernel-api => semantickernel-api-textembedding-services}/src/main/java/com/microsoft/semantickernel/services/AIService.java (100%) rename {semantickernel-api => semantickernel-api-textembedding-services}/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java (100%) rename {semantickernel-api => semantickernel-api-textembedding-services}/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java (100%) rename {semantickernel-api => semantickernel-api-textembedding-services}/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java (100%) diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml new file mode 100644 index 00000000..72421ae0 --- /dev/null +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -0,0 +1,64 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-data-hsqldb + Semantic Kernel HLSQLDB connector + Provides a HLSQLDB connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.apache.commons + commons-text + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java b/data/semantickernel-data-hsqldb/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java rename to data/semantickernel-data-hsqldb/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 5d0883eb..2e9cd67e 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -15,7 +15,7 @@ com.microsoft.semantic-kernel - semantickernel-api + semantickernel-api-data diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index ac21c61f..afc8d82f 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -696,10 +696,19 @@ public String getEqualToFilter(EqualToFilterClause filterClause) { @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider - .validateSQLidentifier(filterClause.getFieldName()); + .validateSQLidentifier(filterClause.getFieldName()); return String.format("%s LIKE ?", fieldName); } + + @Override + public VectorStoreRecordMapper getVectorStoreRecordMapper(Class recordClass, + VectorStoreRecordDefinition recordDefinition) { + return JDBCVectorStoreRecordMapper.builder() + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); + } /** * The builder for {@link JDBCVectorStoreQueryProvider}. @@ -755,4 +764,6 @@ public JDBCVectorStoreQueryProvider build() { prefixForCollectionTables); } } + + } diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java index 1c1cea5d..2d75af14 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -2,9 +2,6 @@ package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.jdbc.mysql.MySQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; @@ -73,25 +70,9 @@ public JDBCVectorStoreRecordCollection( // If mapper is not provided, set a default one if (options.getVectorStoreRecordMapper() == null) { - // Default mapper for PostgreSQL - if (this.queryProvider instanceof PostgreSQLVectorStoreQueryProvider) { - vectorStoreRecordMapper = PostgreSQLVectorStoreRecordMapper.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); - // Default mapper for MySQL - } else if (this.queryProvider instanceof MySQLVectorStoreQueryProvider) { - vectorStoreRecordMapper = JDBCVectorStoreRecordMapper.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); - // Default mapper for other databases - } else { - vectorStoreRecordMapper = JDBCVectorStoreRecordMapper.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); - } + vectorStoreRecordMapper = options.getQueryProvider() + .getVectorStoreRecordMapper(options.getRecordClass(), + recordDefinition); } else { vectorStoreRecordMapper = options.getVectorStoreRecordMapper(); } diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java index 18806c5d..cadfdaf2 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java @@ -151,6 +151,18 @@ VectorSearchResults search(String collectionName, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper); + /** + * Gets the record mapper for the given record class and definition. + * + * @param the record type + * @param recordClass the record class + * @param recordDefinition the record definition + * @return the record mapper that maps JDBC result sets to the given record. + */ + VectorStoreRecordMapper getVectorStoreRecordMapper( + final Class recordClass, + final VectorStoreRecordDefinition recordDefinition); + /** * The builder for the JDBC vector store query provider. */ diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml new file mode 100644 index 00000000..d2d12f13 --- /dev/null +++ b/data/semantickernel-data-mysql/pom.xml @@ -0,0 +1,64 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-data-mysql + Semantic Kernel MySQL connector + Provides a MySQL connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.apache.commons + commons-text + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java b/data/semantickernel-data-mysql/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java rename to data/semantickernel-data-mysql/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml new file mode 100644 index 00000000..e29e758e --- /dev/null +++ b/data/semantickernel-data-postgres/pom.xml @@ -0,0 +1,64 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-data-postgres + Semantic Kernel PostreSQL connector + Provides a PostreSQL connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.apache.commons + commons-text + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java similarity index 97% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java index 4734f484..bd8dbba7 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -442,10 +442,19 @@ public List getFilterParameters(VectorSearchFilter filter) { @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider - .validateSQLidentifier(filterClause.getFieldName()); + .validateSQLidentifier(filterClause.getFieldName()); return String.format("%s @> ?::jsonb", fieldName); } + + @Override + public VectorStoreRecordMapper getVectorStoreRecordMapper(Class recordClass, + VectorStoreRecordDefinition recordDefinition) { + return PostgreSQLVectorStoreRecordMapper.builder() + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); + } /** * A builder for the PostgreSQLVectorStoreQueryProvider class. diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml new file mode 100644 index 00000000..69e3bc6f --- /dev/null +++ b/data/semantickernel-data-sqlite/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + + + com.microsoft.semantic-kernel + semantickernel-data-sqlite + Semantic Kernel SQLite JDBC driver connector + Provides a SQLite connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.apache.commons + commons-text + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java b/data/semantickernel-data-sqlite/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java rename to data/semantickernel-data-sqlite/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java diff --git a/pom.xml b/pom.xml index 8f8ce286..354e87de 100644 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,16 @@ data/semantickernel-data-azureaisearch data/semantickernel-data-jdbc data/semantickernel-data-redis + data/semantickernel-data-mysql + data/semantickernel-data-hsqldb + data/semantickernel-data-postgres + data/semantickernel-data-sqlite agents/semantickernel-agents-core + semantickernel-api-data + semantickernel-api-exceptions + semantickernel-api-builders + semantickernel-api-textembedding-services + semantickernel-api-localization @@ -135,6 +144,31 @@ semantickernel-connectors-ai-openai ${project.version} + + com.microsoft.semantic-kernel + semantickernel-api-builders + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-data + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-localization + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + ${project.version} + com.microsoft.semantic-kernel.extensions semantickernel-sequentialplanner-extension diff --git a/semantickernel-api-builders/pom.xml b/semantickernel-api-builders/pom.xml new file mode 100644 index 00000000..f48a2297 --- /dev/null +++ b/semantickernel-api-builders/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + + + com.microsoft.semantic-kernel + semantickernel-api-builders + Semantic Kernel Builders API + Defines the public interface for the Semantic Kernel Builders + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java b/semantickernel-api-builders/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java rename to semantickernel-api-builders/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java diff --git a/semantickernel-api-data/pom.xml b/semantickernel-api-data/pom.xml new file mode 100644 index 00000000..895e70e1 --- /dev/null +++ b/semantickernel-api-data/pom.xml @@ -0,0 +1,60 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-data + Semantic Kernel Data API + Defines the public interface for the Semantic Kernel Data + + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + io.projectreactor + reactor-core + 3.4.38 + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java diff --git a/semantickernel-api-exceptions/pom.xml b/semantickernel-api-exceptions/pom.xml new file mode 100644 index 00000000..71d1794b --- /dev/null +++ b/semantickernel-api-exceptions/pom.xml @@ -0,0 +1,43 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + Semantic Kernel Exceptions API + Defines the public interface for the Semantic Kernel Exceptions + + + + com.microsoft.semantic-kernel + semantickernel-api-localization + + + com.google.code.findbugs + jsr305 + provided + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java diff --git a/semantickernel-api-localization/pom.xml b/semantickernel-api-localization/pom.xml new file mode 100644 index 00000000..84be7960 --- /dev/null +++ b/semantickernel-api-localization/pom.xml @@ -0,0 +1,31 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-localization + Semantic Kernel Localization API + Defines the public interface for the Semantic Kernel Localization + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java b/semantickernel-api-localization/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java rename to semantickernel-api-localization/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java diff --git a/semantickernel-api/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties b/semantickernel-api-localization/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties similarity index 100% rename from semantickernel-api/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties rename to semantickernel-api-localization/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties diff --git a/semantickernel-api-textembedding-services/pom.xml b/semantickernel-api-textembedding-services/pom.xml new file mode 100644 index 00000000..dbc5fa6a --- /dev/null +++ b/semantickernel-api-textembedding-services/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + Semantic Kernel Services API + Defines the public interface for the Semantic Kernel Services + + + + io.projectreactor + reactor-core + 3.4.38 + + + com.google.code.findbugs + jsr305 + provided + + + com.github.spotbugs + spotbugs-annotations + ${spotbugs.version} + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIService.java b/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/AIService.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIService.java rename to semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/AIService.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java b/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java rename to semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java b/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java rename to semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java b/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java rename to semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index e10ed097..d34e5a17 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -15,6 +15,26 @@ Semantic Kernel API Defines the public interface for the Semantic Kernel + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-localization + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + io.opentelemetry.instrumentation opentelemetry-reactor-3.1 From bf0c167d80296f225aade77a87e89cbff568cfab Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Thu, 22 May 2025 13:18:38 +0200 Subject: [PATCH 04/64] Removed dependencies --- data/semantickernel-data-hsqldb/pom.xml | 22 -------------- data/semantickernel-data-jdbc/pom.xml | 36 +++-------------------- data/semantickernel-data-mysql/pom.xml | 22 -------------- data/semantickernel-data-postgres/pom.xml | 23 ++------------- data/semantickernel-data-sqlite/pom.xml | 23 ++------------- 5 files changed, 10 insertions(+), 116 deletions(-) diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml index 72421ae0..f755f0f1 100644 --- a/data/semantickernel-data-hsqldb/pom.xml +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -24,11 +24,6 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc - - - org.slf4j - slf4j-api - com.fasterxml.jackson.core jackson-databind @@ -39,26 +34,9 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - - org.apache.commons - commons-text - \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 2e9cd67e..7cd77f3f 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -13,15 +13,14 @@ Provides a JDBC connector for the Semantic Kernel - - com.microsoft.semantic-kernel - semantickernel-api-data - - org.slf4j slf4j-api + + com.microsoft.semantic-kernel + semantickernel-api-data + com.fasterxml.jackson.core jackson-databind @@ -32,36 +31,9 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - - org.apache.commons - commons-text - - - org.postgresql - postgresql - 42.7.4 - - - org.xerial - sqlite-jdbc - 3.47.0.0 - \ No newline at end of file diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml index d2d12f13..aefd1cd9 100644 --- a/data/semantickernel-data-mysql/pom.xml +++ b/data/semantickernel-data-mysql/pom.xml @@ -24,11 +24,6 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc - - - org.slf4j - slf4j-api - com.fasterxml.jackson.core jackson-databind @@ -39,26 +34,9 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - - org.apache.commons - commons-text - \ No newline at end of file diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index e29e758e..4382122b 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -24,11 +24,6 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc - - - org.slf4j - slf4j-api - com.fasterxml.jackson.core jackson-databind @@ -39,26 +34,14 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - org.apache.commons - commons-text + org.postgresql + postgresql + 42.7.4 \ No newline at end of file diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index 69e3bc6f..ff4da8b0 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -23,11 +23,6 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc - - - org.slf4j - slf4j-api - com.fasterxml.jackson.core jackson-databind @@ -38,26 +33,14 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - org.apache.commons - commons-text + org.xerial + sqlite-jdbc + 3.47.0.0 \ No newline at end of file From 376d8d215fd5b07a9ed0e7ed0b3d2d08e872c4d6 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 2 Jun 2025 23:31:46 +0200 Subject: [PATCH 05/64] Added oracle store --- data/semantickernel-data-oracle/src/test/resources/initialize.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/test/resources/initialize.sql diff --git a/data/semantickernel-data-oracle/src/test/resources/initialize.sql b/data/semantickernel-data-oracle/src/test/resources/initialize.sql new file mode 100644 index 00000000..e69de29b From 458ace8694d3462b7448dc0757c16cdb4da1b6f3 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 2 Jun 2025 23:33:40 +0200 Subject: [PATCH 06/64] Added Oracle store --- .../jdbc/JDBCVectorStoreQueryProvider.java | 2 +- data/semantickernel-data-oracle/pom.xml | 123 ++++ .../OracleVectorStoreQueryProvider.java | 570 ++++++++++++++++++ .../oracle/OracleVectorStoreRecordMapper.java | 204 +++++++ .../data/jdbc/oracle/Hotel.java | 125 ++++ ...OracleVectorStoreRecordCollectionTest.java | 322 ++++++++++ .../src/test/resources/initialize.sql | 10 + pom.xml | 2 + .../VectorStoreRecordDataField.java | 13 + .../VectorStoreRecordDefinition.java | 3 +- .../definition/VectorStoreRecordField.java | 35 ++ 11 files changed, 1407 insertions(+), 2 deletions(-) create mode 100644 data/semantickernel-data-oracle/pom.xml create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index afc8d82f..9381ef06 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -220,7 +220,7 @@ public Map, String> getSupportedVectorTypes() { @Override public void prepareVectorStore() { String createCollectionsTable = formatQuery( - "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY);", + "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY)", validateSQLidentifier(collectionsTable)); try (Connection connection = dataSource.getConnection(); diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml new file mode 100644 index 00000000..cda12f4f --- /dev/null +++ b/data/semantickernel-data-oracle/pom.xml @@ -0,0 +1,123 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + semantickernel-data-oracle + Semantic Kernel Oracle connector + Provides a Oracle connector for the Semantic Kernel + + + 1.20.4 + + + + + + org.testcontainers + testcontainers-bom + ${testcontainers.version} + pom + import + + + + + + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + provided + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.apache.commons + commons-text + + + org.postgresql + postgresql + 42.7.4 + + + org.xerial + sqlite-jdbc + 3.47.0.0 + + + com.oracle.database.jdbc + ojdbc11 + 23.7.0.25.01 + + + com.oracle.database.jdbc + ojdbc-provider-jackson-oson + 1.0.4 + + + org.junit.jupiter + junit-jupiter + test + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.testcontainers + testcontainers + test + + + org.testcontainers + junit-jupiter + test + + + org.testcontainers + oracle-free + test + + + \ No newline at end of file diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java new file mode 100644 index 00000000..b74226fe --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -0,0 +1,570 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.microsoft.semantickernel.data.jdbc.*; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import oracle.jdbc.OracleResultSet; +import oracle.jdbc.OracleStatement; +import oracle.jdbc.OracleType; +import oracle.jdbc.OracleTypes; + +import javax.annotation.Nonnull; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { + + // This could be removed if super.collectionTable made protected + private final String collectionsTable; + + // This could be common to all query providers + private final ObjectMapper objectMapper; + + private static final Object dbCreationLock = new Object(); + + Logger logger = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); + + public enum StringTypeMapping { + /** + * Maps String to CLOB + */ + USE_CLOB, + /** + * Maps String to VARCHAR2(4000) + */ + USE_VARCHAR + } + + private OracleVectorStoreQueryProvider( + @Nonnull DataSource dataSource, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables, + int defaultVarcharSize, + @Nonnull StringTypeMapping stringTypeMapping, + ObjectMapper objectMapper) { + super( + dataSource, + collectionsTable, + prefixForCollectionTables, + buildSupportedKeyTypes(), + buildSupportedDataTypes(stringTypeMapping, defaultVarcharSize), + buildSupportedVectorTypes(defaultVarcharSize)); + this.collectionsTable = collectionsTable; + this.objectMapper = objectMapper; + } + + private static HashMap, String> buildSupportedKeyTypes() { + HashMap, String> supportedKeyTypes = new HashMap<>(); + supportedKeyTypes.put(String.class, "VARCHAR(255)"); + return supportedKeyTypes; + } + private static Map, String> buildSupportedVectorTypes(int defaultVarCharLength) { + HashMap, String> supportedVectorTypes = new HashMap<>(); + supportedVectorTypes.put(String.class, "VECTOR(%s)"); + supportedVectorTypes.put(List.class, "VECTOR(%s)"); + supportedVectorTypes.put(Collection.class, "VECTOR(%s)"); + return supportedVectorTypes; + } + + private static Map, String> buildSupportedDataTypes(StringTypeMapping stringTypeMapping, int defaultVarCharLength) { + HashMap, String> supportedDataTypes = new HashMap<>(); + if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { + supportedDataTypes.put(String.class, "VARCHAR(" + defaultVarCharLength + ")"); + } else { + supportedDataTypes.put(String.class, "CLOB"); + } + supportedDataTypes.put(Integer.class, "INTEGER"); + supportedDataTypes.put(int.class, "INTEGER"); + supportedDataTypes.put(Long.class, "LONG"); + supportedDataTypes.put(long.class, "LONG"); + supportedDataTypes.put(Float.class, "REAL"); + supportedDataTypes.put(float.class, "REAL"); + supportedDataTypes.put(Double.class, "DOUBLE PRECISION"); + supportedDataTypes.put(double.class, "DOUBLE PRECISION"); + supportedDataTypes.put(Boolean.class, "BOOLEAN"); + supportedDataTypes.put(boolean.class, "BOOLEAN"); + supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); + supportedDataTypes.put(List.class, "JSON"); + return supportedDataTypes; + } + + private String createIndexForVectorField(String collectionName, VectorStoreRecordVectorField vectorField) { + switch (vectorField.getIndexKind()) { + case IVFFLAT: + return "CREATE VECTOR INDEX IF NOT EXISTS " + + getIndexName(vectorField.getEffectiveStorageName()) + + " ON " + + getCollectionTableName(collectionName) + "( " + vectorField.getEffectiveStorageName() + " ) " + + " ORGANIZATION NEIGHBOR PARTITIONS " + + " WITH DISTANCE COSINE " + + "PARAMETERS ( TYPE IVF )"; + case HNSW: + return "CREATE VECTOR INDEX IF NOT EXISTS " + getIndexName(vectorField.getEffectiveStorageName()) + + " ON " + + getCollectionTableName(collectionName) + "( " + vectorField.getEffectiveStorageName() + " ) " + + "ORGANIZATION INMEMORY GRAPH " + + "WITH DISTANCE COSINE " + + "PARAMETERS (TYPE HNSW)"; + case UNDEFINED: + return null; + default: + logger.warning("Unsupported index kind: " + vectorField.getIndexKind()); + return null; + } + } + + private String getIndexName(String effectiveStorageName) { + return effectiveStorageName + "_VECTOR_INDEX"; + } + + + protected String getVectorColumnNamesAndTypes(List fields, + Map, String> types) { + List columns = fields.stream() + .map(field -> validateSQLidentifier(field.getEffectiveStorageName()) + " " + + String.format(types.get(field.getFieldType()), field.getDimensions() > 0 ? field.getDimensions() + ", FLOAT32" : "FLOAT32")) + .collect(Collectors.toList()); + + return String.join(", ", columns); + } + + @Override + protected String getInsertCollectionQuery(String collectionsTable) { + return formatQuery( + "MERGE INTO %s existing "+ + "USING (SELECT ? AS collectionId FROM DUAL) new ON (existing.collectionId = new.collectionId) " + + "WHEN NOT MATCHED THEN INSERT (existing.collectionId) VALUES (new.collectionId)", + collectionsTable); + } + + @Override + public void createCollection(String collectionName, + VectorStoreRecordDefinition recordDefinition) { + + synchronized (dbCreationLock) { + + List vectorFields = recordDefinition.getVectorFields(); + String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" + + "%s VARCHAR(255) PRIMARY KEY, " + + "%s, " + + "%s)", + getCollectionTableName(collectionName), + getKeyColumnName(recordDefinition.getKeyField()), + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), + getSupportedDataTypes()), + getVectorColumnNamesAndTypes(new ArrayList<>(vectorFields), + getSupportedVectorTypes())); + + String insertCollectionQuery = this.getInsertCollectionQuery(collectionsTable); + + try (Connection connection = dataSource.getConnection()) { + connection.createStatement().execute(formatQuery("DROP TABLE IF EXISTS %s", getCollectionTableName(collectionName))); + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + // Create table + System.out.println(createStorageTable); + statement.addBatch(createStorageTable); + + // Index filterable columns + for (VectorStoreRecordDataField dataField : recordDefinition.getDataFields()) { + if (dataField.isFilterable()) { + String dataFieldIndex = createIndexForDataField(collectionName, dataField); + System.out.println(dataFieldIndex); + statement.addBatch(dataFieldIndex); + } + } + + // Create indexed for vectorFields + for (VectorStoreRecordVectorField vectorField : vectorFields) { + String createVectorIndex = createIndexForVectorField(collectionName, + vectorField); + + if (createVectorIndex != null) { + System.out.println(createVectorIndex); + statement.addBatch(createVectorIndex); + } + } + statement.executeBatch(); + + try (PreparedStatement insert = connection.prepareStatement( + insertCollectionQuery)) { + System.out.println(insertCollectionQuery); + insert.setString(1, collectionName); + insert.execute(); + } + + connection.commit(); + } catch (SQLException e) { + connection.rollback(); + throw new SKException("Failed to create collection", e); + } + } catch (SQLException e) { + throw new SKException("Failed to create collection", e); + } + } + } + + private String createIndexForDataField(String collectionName, VectorStoreRecordDataField dataField) { + if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { + String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; + return formatQuery(dataFieldIndex, + getCollectionTableName(collectionName) + "_" + dataField.getEffectiveStorageName(), + getCollectionTableName(collectionName), + dataField.getEffectiveStorageName(), + getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); + } else { + String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; + return formatQuery(dataFieldIndex, + getCollectionTableName(collectionName) + "_" + dataField.getEffectiveStorageName(), + getCollectionTableName(collectionName), + dataField.getEffectiveStorageName() + ); + } + + } + + private String getFunctionForType(String jdbcType) { + switch (jdbcType) { + case "BOOLEAN": + return "boolean()"; + case "INTEGER": + case "LONG": + case "REAL": + case "DOUBLE PRECISION": + return "numberOnly()"; + case "TIMESTAMPTZ": + return "timestamp()"; + default: + return "string()"; + } + } + + @Override + public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + + String upsertQuery = formatQuery("MERGE INTO %s existing "+ + "USING (SELECT %s FROM DUAL) new ON (existing.%s = new.%s) " + + "WHEN MATCHED THEN UPDATE SET %s " + + "WHEN NOT MATCHED THEN INSERT (%s) VALUES (%s)", + getCollectionTableName(collectionName), + getNamedWildcard(recordDefinition.getAllFields()), + getKeyColumnName(recordDefinition.getKeyField()), + getKeyColumnName(recordDefinition.getKeyField()), + getUpdateFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "existing", "new"), + getInsertFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "existing"), + getInsertFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "new")); + + System.out.println(upsertQuery); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(upsertQuery)) { + for (Object record : records) { + setUpsertStatementValues(statement, record, recordDefinition.getAllFields()); + statement.addBatch(); + } + + statement.executeBatch(); + } catch (SQLException e) { + throw new SKException("Failed to upsert records", e); + } + } + + private void setUpsertStatementValues(PreparedStatement statement, Object record, + List fields) { + JsonNode jsonNode = objectMapper.valueToTree(record); + + for (int i = 0; i < fields.size(); ++i) { + VectorStoreRecordField field = fields.get(i); + try { + JsonNode valueNode = jsonNode.get(field.getEffectiveStorageName()); + + if (field instanceof VectorStoreRecordVectorField) { + // Convert the vector field to a string + if (!field.getFieldType().equals(String.class)) { + double[] values = valueNode == null ? null : StreamSupport.stream(((ArrayNode)valueNode).spliterator(), false).mapToDouble(d -> d.asDouble()).toArray(); + statement.setObject(i + 1, values, OracleType.VECTOR_FLOAT64); + System.out.println("Set values: " + values); + continue; + } + } else if (field instanceof VectorStoreRecordDataField) { + // Convert List field to a string + if (field.getFieldType().equals(List.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + System.out.println("Set values: " + objectMapper.writeValueAsString(valueNode)); + continue; + } + } + + statement.setObject(i + 1, + objectMapper.convertValue(valueNode, field.getFieldType())); + System.out.println("Set values: " + objectMapper.convertValue(valueNode, field.getFieldType())); + } catch (SQLException | JsonProcessingException e) { + throw new RuntimeException(e); + } + } + } + private String getInsertFieldList(VectorStoreRecordKeyField key, List fields, String alias) { + return fields.stream().map(f -> alias + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + } + + private String getUpdateFieldList(VectorStoreRecordKeyField key, List fields, String oldAlias, String newAlias) { + return fields.stream().filter(f -> f != key).map(f -> oldAlias + "." + f.getEffectiveStorageName() + " = " + + newAlias + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + + } + + + private String getNamedWildcard(List fields) { + return fields.stream().map(f -> "? " + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + } + + @Override + public VectorSearchResults search(String collectionName, List vector, + VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, + VectorStoreRecordMapper mapper) { + + VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() + .get(0); + VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null + ? firstVectorField + : (VectorStoreRecordVectorField) recordDefinition + .getField(options.getVectorFieldName()); + DistanceFunction distanceFunction = vectorField.getDistanceFunction(); + + List fields; + if (options.isIncludeVectors()) { + fields = recordDefinition.getAllFields(); + } else { + fields = recordDefinition.getNonVectorFields(); + } + + String filter = getFilter(options.getVectorSearchFilter(), recordDefinition); + List parameters = getFilterParameters(options.getVectorSearchFilter()); + + String selectQuery = "SELECT " + + formatQuery("VECTOR_DISTANCE(%s, ?, %s) distance, ", vectorField.getEffectiveStorageName(), toOracleDistanceFunction(distanceFunction)) + + getQueryColumnsFromFields(fields) + + " FROM " + getCollectionTableName(collectionName) + + (filter != null && !filter.isEmpty() ? " WHERE " + filter : "") + + " ORDER BY distance" + + (options.getSkip() > 0 ? " OFFSET " + options.getSkip() + " ROWS" : "") + + (options.getTop() > 0 ? " FETCH " + (options.getSkip() > 0 ? "NEXT " : "FIRST ") + options.getTop() + " ROWS ONLY" : ""); + + System.out.println(selectQuery); + List> records = new ArrayList<>(); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(selectQuery)) { + // set parameters from filters + int parameterIndex = 1; + + statement.setString(parameterIndex++, + objectMapper.writeValueAsString(vector)); + System.out.println("Set vector parameter to: " + objectMapper.writeValueAsString(vector)); + for (Object parameter : parameters) { + statement.setObject(parameterIndex++, parameter); + System.out.println("Set parameter " + parameterIndex + " to: " + parameter); + } + + // Calls to defineColumnType reduce the number of network requests. When Oracle JDBC knows that it is + // fetching VECTOR, CLOB, and/or JSON columns, the first request it sends to the database can include a LOB + // prefetch size (VECTOR and JSON are value-based-lobs). If defineColumnType is not called, then JDBC needs + // to send an additional request with the LOB prefetch size, after the first request has the database + // respond with the column data types. To request all data, the prefetch size is Integer.MAX_VALUE. + OracleStatement oracleStatement = statement.unwrap(OracleStatement.class); + int columnIndex = 1; + defineDataColumnType(columnIndex++, oracleStatement, Double.class); + for (VectorStoreRecordField field : fields) { + if (field instanceof VectorStoreRecordDataField) + defineDataColumnType(columnIndex++, oracleStatement, field.getFieldType()); + else + oracleStatement.defineColumnType(columnIndex++, OracleTypes.VECTOR_FLOAT32, Integer.MAX_VALUE); + } + oracleStatement.setLobPrefetchSize(Integer.MAX_VALUE); // Workaround for Oracle JDBC bug 37030121 + + // get result set + try (ResultSet rs = statement.executeQuery()) { + GetRecordOptions getRecordOptions = new GetRecordOptions(options.isIncludeVectors()); + while (rs.next()) { + // Cosine distance function. 1 - cosine similarity. + double score = Math.abs(rs.getDouble("distance")); + if (distanceFunction == DistanceFunction.COSINE_SIMILARITY) { + score = 1d - score; + } + records.add(new VectorSearchResult<>(mapper.mapStorageModelToRecord(rs, getRecordOptions), score)); + } + } + } catch (SQLException | JsonProcessingException e) { + logger.info(e.getMessage()); + throw new SKException("Search failed", e); + } + + + + return new VectorSearchResults<>(records); + } + + private void defineDataColumnType(int columnIndex, OracleStatement statement, Class fieldType) throws SQLException { + // swich between supported classes and define the column type on the statement + switch (supportedDataTypes.get(fieldType)) { + case "CLOB": + statement.defineColumnType(columnIndex, OracleTypes.CLOB, Integer.MAX_VALUE); + break; + case "INTEGER": + statement.defineColumnType(columnIndex, OracleTypes.INTEGER); + break; + case "LONG": + statement.defineColumnType(columnIndex, OracleTypes.BIGINT); + break; + case "REAL": + statement.defineColumnType(columnIndex, OracleTypes.REAL); + break; + case "DOUBLE PRECISION": + statement.defineColumnType(columnIndex, OracleTypes.BINARY_DOUBLE); + break; + case "BOOLEAN": + statement.defineColumnType(columnIndex, OracleTypes.BOOLEAN); + break; + case "TIMESTAMPTZ": + statement.defineColumnType(columnIndex, OracleTypes.TIMESTAMPTZ); + break; + case "JSON": + statement.defineColumnType(columnIndex, OracleTypes.JSON, Integer.MAX_VALUE); + break; + default: + statement.defineColumnType(columnIndex, OracleTypes.VARCHAR); + } + } + + + private String toOracleDistanceFunction(DistanceFunction distanceFunction) { + switch (distanceFunction) { + case DOT_PRODUCT: + return "DOT"; + case COSINE_SIMILARITY: + case COSINE_DISTANCE: + return "COSINE"; + case EUCLIDEAN_DISTANCE: + return "EUCLIDEAN"; + default: + return "COSINE"; + } + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public VectorStoreRecordMapper getVectorStoreRecordMapper( + Class recordClass, + VectorStoreRecordDefinition vectorStoreRecordDefinition) { + return OracleVectorStoreRecordMapper.builder() + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(vectorStoreRecordDefinition) + .withSupportedDataTypesMapping(getSupportedDataTypes()) + .build(); + } + + public static class Builder + extends JDBCVectorStoreQueryProvider.Builder { + + private DataSource dataSource; + private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + private ObjectMapper objectMapper = new ObjectMapper(); + private StringTypeMapping stringTypeMapping = StringTypeMapping.USE_VARCHAR; + private int defaultVarcharSize = 4000; + + + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; + return this; + } + + /** + * Sets the collections table name. + * @param collectionsTable the collections table name + * @return the builder + */ + public Builder withCollectionsTable(String collectionsTable) { + this.collectionsTable = validateSQLidentifier(collectionsTable); + return this; + } + + /** + * Sets the prefix for collection tables. + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); + return this; + } + + public Builder withObjectMapper( + ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + + /** + * Sets the desired String type mapping. + * @param stringTypeMapping the desired String type mapping. The default value is + * {@link StringTypeMapping#USE_VARCHAR} + * @return the builder + */ + public Builder withStringTypeMapping (StringTypeMapping stringTypeMapping) { + this.stringTypeMapping = stringTypeMapping; + return this; + } + + /** + * Sets the default size of the VARHCHAR2 fields. + * @param defaultVarcharSize the default size of the VARHCHAR2 fields. By default, the size + * is 4000. + * @return then builder + */ + public Builder withDefaultVarcharSize (int defaultVarcharSize) { + this.defaultVarcharSize = defaultVarcharSize; + return this; + } + + @Override + public OracleVectorStoreQueryProvider build() { + return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, + prefixForCollectionTables, defaultVarcharSize, stringTypeMapping, objectMapper); + } + } +} \ No newline at end of file diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java new file mode 100644 index 00000000..1ba9fe7a --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import oracle.jdbc.OracleResultSet; +import oracle.jdbc.provider.oson.OsonModule; +import oracle.sql.json.OracleJsonArray; +import oracle.sql.json.OracleJsonObject; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; + +/** + * Maps a Oracle result set to a record. + * + * @param the record type + */ +public class OracleVectorStoreRecordMapper + extends VectorStoreRecordMapper { + + /** + * Constructs a new instance of the VectorStoreRecordMapper. + * + * @param storageModelToRecordMapper the function to convert a storage model to a record + */ + protected OracleVectorStoreRecordMapper( + BiFunction storageModelToRecordMapper) { + super(null, storageModelToRecordMapper); + } + + /** + * Creates a new builder. + * + * @param the record type + * @return the builder + */ + public static Builder builder() { + return new Builder<>(); + } + + /** + * Operation not supported. + */ + @Override + public ResultSet mapRecordToStorageModel(Record record) { + throw new UnsupportedOperationException("Not implemented"); + } + + /** + * Builder for {@link OracleVectorStoreRecordMapper}. + * + * @param the record type + */ + public static class Builder + implements SemanticKernelBuilder> { + private Class recordClass; + private VectorStoreRecordDefinition vectorStoreRecordDefinition; + private Map, String> supportedDataTypesMapping; + private ObjectMapper objectMapper = new ObjectMapper(); + + /** + * Sets the record class. + * + * @param recordClass the record class + * @return the builder + */ + public Builder withRecordClass(Class recordClass) { + this.recordClass = recordClass; + return this; + } + + /** + * Sets the vector store record definition. + * + * @param vectorStoreRecordDefinition the vector store record definition + * @return the builder + */ + public Builder withVectorStoreRecordDefinition( + VectorStoreRecordDefinition vectorStoreRecordDefinition) { + this.vectorStoreRecordDefinition = vectorStoreRecordDefinition; + return this; + } + + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + + /** + * Sets the Map of supported data types and their database representation + * + * @param supportedDataTypesMapping the Map of supported data types and their + * database representation + * @return the builder + */ + public Builder withSupportedDataTypesMapping( + Map, String> supportedDataTypesMapping) { + this.supportedDataTypesMapping = supportedDataTypesMapping; + return this; + } + + /** + * Builds the {@link OracleVectorStoreRecordMapper}. + * + * @return the {@link OracleVectorStoreRecordMapper} + */ + public OracleVectorStoreRecordMapper build() { + if (recordClass == null) { + throw new SKException("recordClass is required"); + } + if (vectorStoreRecordDefinition == null) { + throw new SKException("vectorStoreRecordDefinition is required"); + } + + return new OracleVectorStoreRecordMapper<>( + (resultSet, options) -> { + try { + objectMapper.registerModule(new OsonModule()); + // Create an ObjectNode to hold the values + ObjectNode objectNode = objectMapper.createObjectNode(); + + // Read non vector fields + for (VectorStoreRecordField field : vectorStoreRecordDefinition.getNonVectorFields()) { + Class fieldType = field.getFieldType(); + + Object value; + switch (supportedDataTypesMapping.get(fieldType)) { + case "CLOB": + value = resultSet.getString(field.getEffectiveStorageName()); + break; + case "INTEGER": + value = resultSet.getInt(field.getEffectiveStorageName()); + break; + case "LONG": + value = resultSet.getInt(field.getEffectiveStorageName()); + break; + case "REAL": + value = resultSet.getFloat(field.getEffectiveStorageName()); + break; + case "DOUBLE PRECISION": + value = resultSet.getDouble(field.getEffectiveStorageName()); + break; + case "BOOLEAN": + value = resultSet.getBoolean(field.getEffectiveStorageName()); + break; + case "TIMESTAMPTZ": + value = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()) + .offsetDateTimeValue(); + break; + case "JSON": + value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); + break; + default: + value = resultSet.getString(field.getEffectiveStorageName()); + } + JsonNode genericNode = objectMapper.valueToTree(value); + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + if (options != null && options.isIncludeVectors()) { + for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + Object value = resultSet.getObject(field.getEffectiveStorageName(), float[].class); + JsonNode genericNode = objectMapper.valueToTree(value); + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + } else { + for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + JsonNode genericNode = objectMapper.valueToTree(null); + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + } + + // Deserialize the object node to the record class + return objectMapper.convertValue(objectNode, recordClass); + } catch (SQLException e) { + throw new SKException( + "Failure to serialize object, by default the JDBC connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", + e); + } + }); + } + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java new file mode 100644 index 00000000..197cb8c3 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; + +import java.util.List; + +import static com.fasterxml.jackson.annotation.JsonCreator.Mode.DELEGATING; +import static com.fasterxml.jackson.annotation.JsonCreator.Mode.PROPERTIES; + +public class Hotel { + @VectorStoreRecordKey + private final String id; + + @VectorStoreRecordData(isFilterable = true) + private final String name; + + @VectorStoreRecordData + private final int code; + + @VectorStoreRecordData + private final double price; + + @VectorStoreRecordData(isFilterable = true) + private final List tags; + + @JsonProperty("summary") + @VectorStoreRecordData( isFilterable = true, isFullTextSearchable = true ) + private final String description; + + @JsonProperty("summaryEmbedding1") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE, indexKind = IndexKind.IVFFLAT) + private final List euclidean; + + @JsonProperty("summaryEmbedding2") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) + private final List cosineDistance; + + @JsonProperty("summaryEmbedding3") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY, indexKind = IndexKind.IVFFLAT) + private final List cosineSimilarity; + + @JsonProperty("summaryEmbedding4") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT, indexKind = IndexKind.IVFFLAT) + private final List dotProduct; + @VectorStoreRecordData + private double rating; + + @JsonCreator(mode = DELEGATING) + public Hotel() { + this(null, null, 0, 0d, null, null, null, null, null, null, 0.0); + } + + @JsonCreator(mode = PROPERTIES) + protected Hotel( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("code") int code, + @JsonProperty("price") double price, + @JsonProperty("tags") List tags, + @JsonProperty("summary") String description, + @JsonProperty("summaryEmbedding1") List euclidean, + @JsonProperty("summaryEmbedding2") List cosineDistance, + @JsonProperty("summaryEmbedding3") List cosineSimilarity, + @JsonProperty("summaryEmbedding4") List dotProduct, + @JsonProperty("rating") double rating) { + this.id = id; + this.name = name; + this.code = code; + this.price = price; + this.tags = tags; + this.description = description; + this.euclidean = euclidean; + this.cosineDistance = euclidean; + this.cosineSimilarity = euclidean; + this.dotProduct = euclidean; + this.rating = rating; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public int getCode() { + return code; + } + + public double getPrice() { return price; } + + public List getTags() { return tags; } + + public String getDescription() { + return description; + } + + public List getEuclidean() { + return euclidean; + } + + public List getCosineDistance() { + return cosineDistance; + } + + public List getDotProduct() { + return dotProduct; + } + + public double getRating() { + return rating; + } + + public void setRating(double rating) { + this.rating = rating; + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java new file mode 100644 index 00000000..546e251f --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -0,0 +1,322 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import oracle.jdbc.OracleConnection; +import oracle.jdbc.datasource.impl.OracleDataSource; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import java.sql.SQLException; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.testcontainers.oracle.OracleContainer; +import org.testcontainers.utility.MountableFile; + +public class OracleVectorStoreRecordCollectionTest { + private static VectorStoreRecordCollection recordCollection; + + private static final String ORACLE_IMAGE_NAME = "gvenzl/oracle-free:23.7-slim-faststart"; + private static final OracleDataSource DATA_SOURCE; + private static final OracleDataSource SYSDBA_DATA_SOURCE; + + + static { + + try { + DATA_SOURCE = new oracle.jdbc.datasource.impl.OracleDataSource(); + SYSDBA_DATA_SOURCE = new oracle.jdbc.datasource.impl.OracleDataSource(); + String urlFromEnv = System.getenv("ORACLE_JDBC_URL"); + + if (urlFromEnv == null) { + // The Ryuk component is relied upon to stop this container. + OracleContainer oracleContainer = new OracleContainer(ORACLE_IMAGE_NAME) + .withCopyFileToContainer(MountableFile.forClasspathResource("/initialize.sql"), + "/container-entrypoint-initdb.d/initialize.sql") + .withStartupTimeout(Duration.ofSeconds(600)) + .withConnectTimeoutSeconds(600) + .withDatabaseName("pdb1") + .withUsername("testuser") + .withPassword("testpwd"); + oracleContainer.start(); + + initDataSource( + DATA_SOURCE, + oracleContainer.getJdbcUrl(), + oracleContainer.getUsername(), + oracleContainer.getPassword()); + initDataSource(SYSDBA_DATA_SOURCE, oracleContainer.getJdbcUrl(), "sys", oracleContainer.getPassword()); + } else { + initDataSource( + DATA_SOURCE, + urlFromEnv, + System.getenv("ORACLE_JDBC_USER"), + System.getenv("ORACLE_JDBC_PASSWORD")); + initDataSource( + SYSDBA_DATA_SOURCE, + urlFromEnv, + System.getenv("ORACLE_JDBC_USER"), + System.getenv("ORACLE_JDBC_PASSWORD")); + } + SYSDBA_DATA_SOURCE.setConnectionProperty(OracleConnection.CONNECTION_PROPERTY_INTERNAL_LOGON, "SYSDBA"); + + } catch (SQLException sqlException) { + throw new AssertionError(sqlException); + } + } + + static void initDataSource(OracleDataSource dataSource, String url, String username, String password) { + dataSource.setURL(url); + dataSource.setUser(username); + dataSource.setPassword(password); + } + + @BeforeAll + public static void setup() throws Exception { + + // Build a query provider + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + // Build a vector store + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + // Get a collection from the vector store + recordCollection = + vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + recordCollection.createCollectionIfNotExistsAsync().block(); + } + + @BeforeEach + public void clearCollection() { + recordCollection.deleteCollectionAsync().block(); + recordCollection.createCollectionAsync().block(); + } + + private static List getHotels() { + return Arrays.asList( + new Hotel("id_1", "Hotel 1", 1, 1.49d, null, "Hotel 1 description", + Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + 4.0), + new Hotel("id_2", "Hotel 2", 2, 1.44d, null, "Hotel 2 description with free-text search", + Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, + 4.0), + new Hotel("id_3", "Hotel 3", 3, 1.53d, null, "Hotel 3 description", + Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, + 5.0), + new Hotel("id_4", "Hotel 4", 4, 1.35d, null, "Hotel 4 description", + Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, + 4.0), + new Hotel("id_5", "Hotel 5", 5, 1.89d, null,"Hotel 5 description", + Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + 4.0)); + } + + /** + * Search embeddings similar to the third hotel embeddings. + * In order of similarity: + * 1. Hotel 3 + * 2. Hotel 1 + * 3. Hotel 4 + */ + private static final List SEARCH_EMBEDDINGS = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, + -0.8f, 1.1f, -2.2f, 8.2f); + + @Test + public void createAndDeleteCollectionAsync() { + assertEquals(true, recordCollection.collectionExistsAsync().block()); + + recordCollection.deleteCollectionAsync().block(); + assertEquals(false, recordCollection.collectionExistsAsync().block()); + + recordCollection.createCollectionAsync().block(); + assertEquals(true, recordCollection.collectionExistsAsync().block()); + } + + @Test + public void upsertRecordAsync() { + List hotels = getHotels(); + for (Hotel hotel : hotels) { + recordCollection.upsertAsync(hotel, null).block(); + } + + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + assertNotNull(retrievedHotel); + assertEquals(hotel.getId(), retrievedHotel.getId()); + assertEquals(hotel.getName(), retrievedHotel.getName()); + assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); + } + } + + @Test + public void upsertBatchAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + assertNotNull(retrievedHotel); + assertEquals(hotel.getId(), retrievedHotel.getId()); + assertEquals(hotel.getName(), retrievedHotel.getName()); + assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); + } + } + + @Test + public void getBatchAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + List keys = hotels.stream().map(Hotel::getId).collect(Collectors.toList()); + List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); + + assertNotNull(retrievedHotels); + assertEquals(keys.size(), retrievedHotels.size()); + for (Hotel hotel : retrievedHotels) { + assertTrue(keys.contains(hotel.getId())); + } + } + + @Test + public void deleteRecordAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + for (Hotel hotel : hotels) { + recordCollection.deleteAsync(hotel.getId(), null).block(); + assertNull(recordCollection.getAsync(hotel.getId(), null).block()); + } + } + + @Test + public void deleteBatchAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + List keys = hotels.stream().map(Hotel::getId).collect(Collectors.toList()); + recordCollection.deleteBatchAsync(keys, null).block(); + + for (String key : keys) { + assertNull(recordCollection.getAsync(key, null).block()); + } + } + + @ParameterizedTest + @MethodSource("parametersExactSearch") + public void exactSearch(DistanceFunction distanceFunction, List expectedDistance) { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withTop(3) + .build(); + + // Embeddings similar to the third hotel + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(3, results.size()); + // The third hotel should be the most similar + System.out.println(results.get(0).getScore()); + System.out.println(results.get(1).getScore()); + System.out.println(results.get(2).getScore()); + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertEquals(expectedDistance.get(0).doubleValue(), results.get(0).getScore(), 0.0001d); + assertEquals(hotels.get(0).getId(), results.get(1).getRecord().getId()); + assertEquals(expectedDistance.get(1).doubleValue(), results.get(1).getScore(), 0.0001d); + assertEquals(hotels.get(3).getId(), results.get(2).getRecord().getId()); + assertEquals(expectedDistance.get(2).doubleValue(), results.get(2).getScore(), 0.0001d); + + options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withSkip(1) + .withTop(-100) + .build(); + + // Skip the first result + results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(1, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + assertEquals(results.get(0).getScore(), expectedDistance.get(1), 0.001d); + } + + @ParameterizedTest + @MethodSource("distanceFunctionAndDistance") + public void searchWithFilter(DistanceFunction distanceFunction, double expectedDistance) { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withTop(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("rating", 4.0).build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(3, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + assertEquals(results.get(0).getScore(), expectedDistance, 0.0001d); + } + + private static Stream distanceFunctionAndDistance() { + return Stream.of( + Arguments.of (DistanceFunction.COSINE_DISTANCE, 0.8548d), + Arguments.of (DistanceFunction.COSINE_SIMILARITY, 0.1451d), + Arguments.of (DistanceFunction.DOT_PRODUCT, 30.3399d), + Arguments.of (DistanceFunction.EUCLIDEAN_DISTANCE, 18.9081d), + Arguments.of (DistanceFunction.UNDEFINED, 18.9081d) + ); + } + + private static Stream parametersExactSearch() { + return Stream.of( + Arguments.of (DistanceFunction.COSINE_SIMILARITY, Arrays.asList(0.9999d, 0.1451d, 0.0178d)), + Arguments.of (DistanceFunction.COSINE_DISTANCE, Arrays.asList(1.6422E-5d, 0.8548d, 0.9821d)), + Arguments.of (DistanceFunction.DOT_PRODUCT, Arrays.asList(202.3399d, 30.3399d, 3.6199d)), + Arguments.of (DistanceFunction.EUCLIDEAN_DISTANCE, Arrays.asList(0.1000d, 18.9081d, 19.9669d)), + Arguments.of (DistanceFunction.UNDEFINED, Arrays.asList(0.1000d, 18.9081d, 19.9669d)) + ); + } +} diff --git a/data/semantickernel-data-oracle/src/test/resources/initialize.sql b/data/semantickernel-data-oracle/src/test/resources/initialize.sql index e69de29b..4b065473 100644 --- a/data/semantickernel-data-oracle/src/test/resources/initialize.sql +++ b/data/semantickernel-data-oracle/src/test/resources/initialize.sql @@ -0,0 +1,10 @@ +-- Exit on any errors +WHENEVER SQLERROR EXIT SQL.SQLCODEAdd commentMore actions + +-- Configure the size of the Vector Pool to 1 GiB. +ALTER SYSTEM SET vector_memory_size=1G SCOPE=SPFILE; + +SHUTDOWN ABORT; +STARTUP; + +exit; \ No newline at end of file diff --git a/pom.xml b/pom.xml index 354e87de..b52f7484 100644 --- a/pom.xml +++ b/pom.xml @@ -81,6 +81,7 @@ data/semantickernel-data-hsqldb data/semantickernel-data-postgres data/semantickernel-data-sqlite + data/semantickernel-data-oracle agents/semantickernel-agents-core semantickernel-api-data semantickernel-api-exceptions @@ -249,6 +250,7 @@ ${maven.compiler.release} ${maven.compiler.release} 8 + -Xlint:unchecked diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java index 9e5aea11..713cae29 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java @@ -39,6 +39,18 @@ public VectorStoreRecordDataField( this.isFullTextSearchable = isFullTextSearchable; } + public VectorStoreRecordDataField( + @Nonnull String name, + @Nullable String storageName, + @Nonnull Class fieldType, + @Nonnull Class fieldSubType, + boolean isFilterable, + boolean isFullTextSearchable) { + super(name, storageName, fieldType, fieldSubType); + this.isFilterable = isFilterable; + this.isFullTextSearchable = isFullTextSearchable; + } + /** * Gets a value indicating whether the field is filterable. * @@ -105,6 +117,7 @@ public VectorStoreRecordDataField build() { name, storageName, fieldType, + fieldSubType, isFilterable, isFullTextSearchable); } diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 54b2bf2b..8f8d6212 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -7,6 +7,7 @@ import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.exceptions.SKException; import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -193,7 +194,7 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) dataFields.add(VectorStoreRecordDataField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType()) + .withFieldType(field.getType(), List.class.equals(field.getType()) ? (Class)((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0] : null) .isFilterable(dataAttribute.isFilterable()) .build()); } diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java index 0ba377af..f777bd5c 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java @@ -14,6 +14,7 @@ public class VectorStoreRecordField { @Nullable private final String storageName; private final Class fieldType; + private final Class fieldSubType; /** * Creates a new instance of the VectorStoreRecordField class. @@ -29,6 +30,27 @@ public VectorStoreRecordField( this.name = name; this.storageName = storageName; this.fieldType = fieldType; + this.fieldSubType = null; + } + + /** + * Creates a new instance of the VectorStoreRecordField class. + * + * @param name the name of the field + * @param storageName the storage name of the field + * @param fieldType the field type + * @param fieldSubType if the field type is a list, the type of + * the list elements, otherwise null + */ + public VectorStoreRecordField( + @Nonnull String name, + @Nullable String storageName, + @Nonnull Class fieldType, + @Nonnull Class fieldSubType) { + this.name = name; + this.storageName = storageName; + this.fieldType = fieldType; + this.fieldSubType = fieldSubType; } /** @@ -68,6 +90,10 @@ public Class getFieldType() { return fieldType; } + public Class getFieldSubType() { + return fieldSubType; + } + /** * A builder for the VectorStoreRecordField class. * @param the type of the field @@ -83,6 +109,9 @@ public abstract static class Builder> @Nullable protected Class fieldType; + @Nullable + protected Class fieldSubType; + /** * Sets the name of the field. * @@ -116,6 +145,12 @@ public U withFieldType(Class fieldType) { return (U) this; } + public U withFieldType(Class fieldType, Class fieldSubType) { + this.fieldType = fieldType; + this.fieldSubType = fieldSubType; + return (U) this; + } + /** * Builds the field. * From 8bef1ffad35925c95059e0ba7ea24439847e220f Mon Sep 17 00:00:00 2001 From: psilberk Date: Mon, 2 Jun 2025 16:14:02 -0700 Subject: [PATCH 07/64] Added reference to parent's pom in sqlite --- data/semantickernel-data-sqlite/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index ff4da8b0..c8a747a9 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -7,6 +7,7 @@ com.microsoft.semantic-kernel semantickernel-parent 1.4.4-RC2-SNAPSHOT + ../../pom.xml com.microsoft.semantic-kernel From a514b289a7b663b6a87074db5ba9dfc7abbfd10f Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 3 Jun 2025 17:01:00 +0200 Subject: [PATCH 08/64] Any tag filter --- .../OracleVectorStoreQueryProvider.java | 47 +++++++++++++++++-- ...OracleVectorStoreRecordCollectionTest.java | 34 ++++++++++++-- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index b74226fe..b217b0e4 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -4,7 +4,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.jdbc.*; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; @@ -34,6 +37,7 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -387,13 +391,12 @@ public VectorSearchResults search(String collectionName, List getFilterParameters(VectorSearchFilter filter) { + if (filter == null + || filter.getFilterClauses().isEmpty()) { + return Collections.emptyList(); + } + + return filter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return equalToFilterClause.getValue(); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + return anyTagEqualToFilterClause.getValue(); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.toList()); + } + + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + + return String.format("JSON_EXISTS(%s, '$[*]?(@ == $v_%s)' PASSING ? AS \"v_%s\")", + fieldName, fieldName, fieldName); + } + + public static Builder builder() { return new Builder(); } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 546e251f..72ed3524 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -127,19 +127,19 @@ public void clearCollection() { private static List getHotels() { return Arrays.asList( - new Hotel("id_1", "Hotel 1", 1, 1.49d, null, "Hotel 1 description", + new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, 4.0), - new Hotel("id_2", "Hotel 2", 2, 1.44d, null, "Hotel 2 description with free-text search", + new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, 4.0), - new Hotel("id_3", "Hotel 3", 3, 1.53d, null, "Hotel 3 description", + new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, 5.0), - new Hotel("id_4", "Hotel 4", 4, 1.35d, null, "Hotel 4 description", + new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, 4.0), - new Hotel("id_5", "Hotel 5", 5, 1.89d, null,"Hotel 5 description", + new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, 4.0)); } @@ -300,6 +300,30 @@ public void searchWithFilter(DistanceFunction distanceFunction, double expectedD assertEquals(results.get(0).getScore(), expectedDistance, 0.0001d); } + + @Test + public void searchWithTagFilter() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() +// .withVectorFieldName("") + .withTop(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .anyTagEqualTo("tags", "three") + .build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(1, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); + } + private static Stream distanceFunctionAndDistance() { return Stream.of( Arguments.of (DistanceFunction.COSINE_DISTANCE, 0.8548d), From d8b6895931dac5ae418622ecb4222f14fac4fc62 Mon Sep 17 00:00:00 2001 From: psilberk Date: Thu, 5 Jun 2025 15:29:10 -0700 Subject: [PATCH 09/64] Clean pom and 1st Sample --- data/semantickernel-data-oracle/pom.xml | 40 ----------------- .../src/test/resources/initialize.sql | 4 +- .../data/vectorstores/oracle/Main.java | 43 +++++++++++++++++++ 3 files changed, 46 insertions(+), 41 deletions(-) create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index cda12f4f..d6218548 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -29,18 +29,10 @@ - - com.microsoft.semantic-kernel - semantickernel-api-data - com.microsoft.semantic-kernel semantickernel-data-jdbc - - org.slf4j - slf4j-api - com.fasterxml.jackson.core jackson-databind @@ -51,38 +43,6 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - - - com.github.spotbugs - spotbugs-annotations - - - org.apache.commons - commons-text - - - org.postgresql - postgresql - 42.7.4 - - - org.xerial - sqlite-jdbc - 3.47.0.0 - com.oracle.database.jdbc ojdbc11 diff --git a/data/semantickernel-data-oracle/src/test/resources/initialize.sql b/data/semantickernel-data-oracle/src/test/resources/initialize.sql index 4b065473..8756f121 100644 --- a/data/semantickernel-data-oracle/src/test/resources/initialize.sql +++ b/data/semantickernel-data-oracle/src/test/resources/initialize.sql @@ -4,7 +4,9 @@ WHENEVER SQLERROR EXIT SQL.SQLCODEAdd commentMore actions -- Configure the size of the Vector Pool to 1 GiB. ALTER SYSTEM SET vector_memory_size=1G SCOPE=SPFILE; +sqlplus / as sysdba + SHUTDOWN ABORT; STARTUP; -exit; \ No newline at end of file +exit \ No newline at end of file diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java new file mode 100644 index 00000000..950a62f3 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; + +public class Main { + public static void main(String[] args) { + // Configure the data source + OracleDataSource dataSource = new OracleDataSource(); + dataSource.setURL("jdbc:oracle:thin:@localhost:1521/FREEPDB1"); + dataSource.setUser("scott"); + dataSource.setPassword("tiger"); + + // Build a query provider + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + + // Build a vector store + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = vectorStore.getCollection( + "skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + // Create the collection if it doesn't exist yet. + collection.createCollectionIfNotExistsAsync().block(); + + collection.upsertBatchAsync(getHotels(), null).block(); + } +} From df07b40a9d09f91acc0e0633e0f3493670d03687 Mon Sep 17 00:00:00 2001 From: psilberk Date: Thu, 5 Jun 2025 15:41:19 -0700 Subject: [PATCH 10/64] Sample wip #2 --- .../data/vectorstores/oracle/Main.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java index 2c172bd4..fe37be8a 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -39,5 +39,40 @@ public static void main(String[] args) { collection.createCollectionIfNotExistsAsync().block(); collection.upsertBatchAsync(getHotels(), null).block(); + + // Retrieve the upserted record. + var retrievedHotel = collection.getAsync("1", null).block(); + + // Generate a vector for your search text, using your chosen embedding generation implementation. + // Just showing a placeholder method here for brevity. + // var searchVector = generateEmbeddingsAsync( + // "I'm looking for a hotel where customer happiness is the priority.").block(); + + // Do the search. + // var searchResult = collection.searchAsync(searchVector, VectorSearchOptions.builder() + // .withTop(1).build()).block(); + + // Hotel record = searchResult.getResults().get(0).getRecord(); + // System.out.printf("Found hotel description: %s\n", record.getDescription()); + + } + + private static List getHotels() { + return Arrays.asList( + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", + Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", + Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, + 4.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", + Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, + 5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", + Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, + 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", + Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + 4.0)); } } \ No newline at end of file From 805f5fa8d55a8236972715d1f87eb457c1e89c1f Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 7 Jun 2025 00:52:21 -0700 Subject: [PATCH 11/64] Add test for IndexKind.UNDEFINED and IndexKind.HNSW --- ...OracleVectorStoreRecordCollectionTest.java | 161 +++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 72ed3524..4ec1599d 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -9,17 +9,28 @@ import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import oracle.jdbc.OracleConnection; import oracle.jdbc.datasource.impl.OracleDataSource; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.time.Duration; import java.util.Arrays; import java.util.List; @@ -42,7 +53,6 @@ public class OracleVectorStoreRecordCollectionTest { private static final OracleDataSource DATA_SOURCE; private static final OracleDataSource SYSDBA_DATA_SOURCE; - static { try { @@ -324,6 +334,155 @@ public void searchWithTagFilter() { assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); } + @Nested + class HNSWIndexTests { + @Test + void testHNSWIndexIsCreatedSuccessfully() throws Exception { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .isFilterable(false) + .build(); + + VectorStoreRecordVectorField hnswVector= VectorStoreRecordVectorField.builder() + .withName("hnsw") + .withStorageName("hnsw") + .withFieldType(List.class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_SIMILARITY) + .withIndexKind(IndexKind.HNSW) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, hnswVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "skhotels_hnsw"; + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Object.class) + .withRecordDefinition(definition) + .build()); + + // create collection + collection.createCollectionAsync().block(); + + String expectedIndexName = hnswVector.getEffectiveStorageName().toUpperCase() + "_VECTOR_INDEX"; + + // check if index exist + try (Connection conn = DATA_SOURCE.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "SELECT COUNT(*) FROM USER_INDEXES WHERE INDEX_NAME=?")) { + stmt.setString(1, expectedIndexName); + ResultSet rs = stmt.executeQuery(); + rs.next(); + int count = rs.getInt(1); + + assertEquals(1, count, "hnsw vector index should have been created"); + } finally { + // clean up + try (Connection conn = DATA_SOURCE.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE " + "SKCOLLECTION_" + collectionName); + } + } + } + } + + @Nested + class UndefinedIndexTests { + @Test + void testNoIndexIsCreatedForUndefined() throws Exception { + // create key field + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + // create vector field, set IndexKind to UNDEFINED + VectorStoreRecordVectorField undefinedVector= VectorStoreRecordVectorField.builder() + .withName("undef") + .withStorageName("undef") + .withFieldType(List.class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_SIMILARITY) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .isFilterable(false) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, undefinedVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "skhotels_undefined"; + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Object.class) + .withRecordDefinition(definition) + .build()); + + // create collection + collection.createCollectionAsync().block(); + + // check if index exist + String expectedIndexName = undefinedVector.getEffectiveStorageName().toUpperCase() + "_VETCOR_INDEX"; + try (Connection conn = DATA_SOURCE.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "SELECT COUNT(*) FROM USER_INDEXES WHERE INDEX_NAME = ?")) { + stmt.setString(1, expectedIndexName); + ResultSet rs = stmt.executeQuery(); + rs.next(); + int count = rs.getInt(1); + + assertEquals(0,count,"Vector index should not be created for IndexKind.UNDEFINED"); + } finally { + // clean up + try (Connection conn = DATA_SOURCE.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE " + "SKCOLLECTION_" + collectionName); + } + } + } + } + private static Stream distanceFunctionAndDistance() { return Stream.of( Arguments.of (DistanceFunction.COSINE_DISTANCE, 0.8548d), From 1ca94390fe2078fbb0bdecfef2dc647e5bb1fc5a Mon Sep 17 00:00:00 2001 From: psilberk Date: Mon, 9 Jun 2025 15:22:43 -0700 Subject: [PATCH 12/64] Remove old implementation --- .../OracleVectorStoreQueryProvider.java | 144 ------------------ 1 file changed, 144 deletions(-) delete mode 100644 data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java deleted file mode 100644 index 10af511c..00000000 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ /dev/null @@ -1,144 +0,0 @@ -package com.microsoft.semantickernel.data.jdbc.oracle; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; -import com.microsoft.semantickernel.exceptions.SKException; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - -import javax.annotation.Nonnull; -import javax.sql.DataSource; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.List; - -public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { - - // This could be removed if super.collectionTable made protected - private final String collectionsTable; - - // This could be common to all query providers - private final ObjectMapper objectMapper; - - private OracleVectorStoreQueryProvider(@Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables, - ObjectMapper objectMapper) { - super(dataSource, collectionsTable, prefixForCollectionTables); - this.collectionsTable = collectionsTable; - this.objectMapper = objectMapper; - } - - @Override - public void prepareVectorStore() { - String createCollectionsTable = formatQuery( - "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY)", - validateSQLidentifier(collectionsTable)); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement createTable = connection.prepareStatement(createCollectionsTable)) { - createTable.execute(); - } catch (SQLException e) { - throw new SKException("Failed to prepare vector store", e); - } - } - - @Override - public void createCollection(String collectionName, - VectorStoreRecordDefinition recordDefinition) { - // TODO Override implementation. Eg: mapping TEXT to VARCHAR - super.createCollection(collectionName, recordDefinition); - } - - @Override - public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { - - // TODO look for public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { - - // TODO Make this a MERGE query - -// String upsertStatemente = formatQuery(""" -// MERGE INTO %s EXIST_REC USING (SELECT ? AS ID) NEW_REC ON (EXIST_REC.%s = NEW_REC.ID) -// WHEN MATACHED THEN UPDATE SET EXISTING REC -// """, -// getCollectionTableName(collectionName), -// recordDefinition.getKeyField().getName(), -// getQueryColumnsFromFields(fields), -// getWildcardString(fields.size()), -// onDuplicateKeyUpdate);super.upsertRecords(collectionName, records, recordDefinition, options); - - String query = formatQuery("INSERT INTO %s (%s, %s, %s) values (?, ?, ?)", - getCollectionTableName(collectionName), - recordDefinition.getAllFields().get(0).getStorageName(), - recordDefinition.getAllFields().get(1).getStorageName(), - recordDefinition.getAllFields().get(2).getStorageName()); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(query)) { - for (Object record : records) { - JsonNode jsonNode = objectMapper.valueToTree(record); - for (int i = 0; i < 3; i++) { - statement.setObject(i + 1, jsonNode - .get(recordDefinition.getAllFields().get(i).getStorageName()).asText()); - } - statement.addBatch(); - } - statement.executeBatch(); - } catch (SQLException e) { - throw new SKException("Failed to upsert records", e); - } - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder - extends JDBCVectorStoreQueryProvider.Builder { - - private DataSource dataSource; - private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; - private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; - private ObjectMapper objectMapper = new ObjectMapper(); - - @SuppressFBWarnings("EI_EXPOSE_REP2") - public Builder withDataSource(DataSource dataSource) { - this.dataSource = dataSource; - return this; - } - - /** - * Sets the collections table name. - * @param collectionsTable the collections table name - * @return the builder - */ - public Builder withCollectionsTable(String collectionsTable) { - this.collectionsTable = validateSQLidentifier(collectionsTable); - return this; - } - - /** - * Sets the prefix for collection tables. - * @param prefixForCollectionTables the prefix for collection tables - * @return the builder - */ - public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { - this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); - return this; - } - - public Builder withObjectMapper( - ObjectMapper objectMapper) { - this.objectMapper = objectMapper; - return this; - } - - @Override - public OracleVectorStoreQueryProvider build() { - return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, - prefixForCollectionTables, objectMapper); - } - } -} \ No newline at end of file From c3bfafbd6109782c7c452c82345f82559d6d38ea Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 10 Jun 2025 17:55:16 +0200 Subject: [PATCH 13/64] Added constants for type mapping and matched ODP type mapping when possible. --- .../jdbc/oracle/OracleDataTypesMapping.java | 18 ++++++ .../OracleVectorStoreQueryProvider.java | 64 +++++++++++++------ .../oracle/OracleVectorStoreRecordMapper.java | 32 +++++++--- 3 files changed, 84 insertions(+), 30 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java new file mode 100644 index 00000000..f441a440 --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -0,0 +1,18 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +public class OracleDataTypesMapping { + public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; + public static final String STRING_CLOB = "CLOB"; + public static final String BOOLEAN = "BOOLEAN"; + public static final String BYTE = "NUMBER(3)"; + public static final String BYTE_ARRAY = "RAW(2000)"; + public static final String SHORT = "NUMBER(5)"; + public static final String INTEGER = "NUMBER(10)"; + public static final String LONG = "NUMBER(19)"; + public static final String FLOAT = "BINARY_FLOAT"; + public static final String DOUBLE = "BINARY_DOUBLE"; + public static final String DECIMAL = "NUMBER(18,2)"; + public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; + public static final String UUID = "RAW(16)"; + public static final String JSON = "JSON"; +} diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index b217b0e4..12466a6f 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -29,6 +29,7 @@ import javax.annotation.Nonnull; import javax.sql.DataSource; +import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -41,6 +42,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -81,17 +83,25 @@ private OracleVectorStoreQueryProvider( prefixForCollectionTables, buildSupportedKeyTypes(), buildSupportedDataTypes(stringTypeMapping, defaultVarcharSize), - buildSupportedVectorTypes(defaultVarcharSize)); + buildSupportedVectorTypes()); this.collectionsTable = collectionsTable; this.objectMapper = objectMapper; } private static HashMap, String> buildSupportedKeyTypes() { HashMap, String> supportedKeyTypes = new HashMap<>(); - supportedKeyTypes.put(String.class, "VARCHAR(255)"); + supportedKeyTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); + supportedKeyTypes.put(short.class, OracleDataTypesMapping.SHORT); + supportedKeyTypes.put(Short.class, OracleDataTypesMapping.SHORT); + supportedKeyTypes.put(int.class, OracleDataTypesMapping.INTEGER); + supportedKeyTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); + supportedKeyTypes.put(long.class, OracleDataTypesMapping.LONG); + supportedKeyTypes.put(Long.class, OracleDataTypesMapping.LONG); + supportedKeyTypes.put(UUID.class, OracleDataTypesMapping.UUID); + return supportedKeyTypes; } - private static Map, String> buildSupportedVectorTypes(int defaultVarCharLength) { + private static Map, String> buildSupportedVectorTypes() { HashMap, String> supportedVectorTypes = new HashMap<>(); supportedVectorTypes.put(String.class, "VECTOR(%s)"); supportedVectorTypes.put(List.class, "VECTOR(%s)"); @@ -102,22 +112,29 @@ private static Map, String> buildSupportedVectorTypes(int defaultVarCha private static Map, String> buildSupportedDataTypes(StringTypeMapping stringTypeMapping, int defaultVarCharLength) { HashMap, String> supportedDataTypes = new HashMap<>(); if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { - supportedDataTypes.put(String.class, "VARCHAR(" + defaultVarCharLength + ")"); + supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); } else { - supportedDataTypes.put(String.class, "CLOB"); + supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); } - supportedDataTypes.put(Integer.class, "INTEGER"); - supportedDataTypes.put(int.class, "INTEGER"); - supportedDataTypes.put(Long.class, "LONG"); - supportedDataTypes.put(long.class, "LONG"); - supportedDataTypes.put(Float.class, "REAL"); - supportedDataTypes.put(float.class, "REAL"); - supportedDataTypes.put(Double.class, "DOUBLE PRECISION"); - supportedDataTypes.put(double.class, "DOUBLE PRECISION"); - supportedDataTypes.put(Boolean.class, "BOOLEAN"); - supportedDataTypes.put(boolean.class, "BOOLEAN"); - supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); - supportedDataTypes.put(List.class, "JSON"); + supportedDataTypes.put(byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(Byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(Short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(int.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(Double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); + supportedDataTypes.put(Boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); + supportedDataTypes.put(UUID.class, OracleDataTypesMapping.UUID); + supportedDataTypes.put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); + supportedDataTypes.put(List.class, OracleDataTypesMapping.JSON); return supportedDataTypes; } @@ -178,11 +195,11 @@ public void createCollection(String collectionName, List vectorFields = recordDefinition.getVectorFields(); String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" - + "%s VARCHAR(255) PRIMARY KEY, " + + "%s PRIMARY KEY, " + "%s, " + "%s)", getCollectionTableName(collectionName), - getKeyColumnName(recordDefinition.getKeyField()), + getKeyColumnNameAndType(recordDefinition.getKeyField(), getSupportedDataTypes()), getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), getSupportedDataTypes()), getVectorColumnNamesAndTypes(new ArrayList<>(vectorFields), @@ -237,6 +254,11 @@ public void createCollection(String collectionName, } } + private String getKeyColumnNameAndType(VectorStoreRecordKeyField field, Map, String> types) { + return validateSQLidentifier(field.getEffectiveStorageName()) + " " + + types.get(field.getFieldType()); + } + private String createIndexForDataField(String collectionName, VectorStoreRecordDataField dataField) { if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; @@ -545,7 +567,7 @@ public static class Builder private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; private ObjectMapper objectMapper = new ObjectMapper(); private StringTypeMapping stringTypeMapping = StringTypeMapping.USE_VARCHAR; - private int defaultVarcharSize = 4000; + private int defaultVarcharSize = 2000; @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -594,7 +616,7 @@ public Builder withStringTypeMapping (StringTypeMapping stringTypeMapping) { /** * Sets the default size of the VARHCHAR2 fields. * @param defaultVarcharSize the default size of the VARHCHAR2 fields. By default, the size - * is 4000. + * is 2000. * @return then builder */ public Builder withDefaultVarcharSize (int defaultVarcharSize) { diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 1ba9fe7a..eaa8e7dc 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -147,29 +147,43 @@ public OracleVectorStoreRecordMapper build() { Object value; switch (supportedDataTypesMapping.get(fieldType)) { - case "CLOB": + case OracleDataTypesMapping.STRING_CLOB: value = resultSet.getString(field.getEffectiveStorageName()); break; - case "INTEGER": - value = resultSet.getInt(field.getEffectiveStorageName()); + case OracleDataTypesMapping.BYTE: + value = resultSet.getByte(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.SHORT: + value = resultSet.getShort(field.getEffectiveStorageName()); break; - case "LONG": + case OracleDataTypesMapping.INTEGER: value = resultSet.getInt(field.getEffectiveStorageName()); break; - case "REAL": + case OracleDataTypesMapping.LONG: + value = resultSet.getLong(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.FLOAT: value = resultSet.getFloat(field.getEffectiveStorageName()); break; - case "DOUBLE PRECISION": + case OracleDataTypesMapping.DOUBLE: value = resultSet.getDouble(field.getEffectiveStorageName()); break; - case "BOOLEAN": + case OracleDataTypesMapping.DECIMAL: + value = resultSet.getBigDecimal(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.BOOLEAN: value = resultSet.getBoolean(field.getEffectiveStorageName()); break; - case "TIMESTAMPTZ": + case OracleDataTypesMapping.OFFSET_DATE_TIME: value = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()) .offsetDateTimeValue(); break; - case "JSON": + case OracleDataTypesMapping.BYTE_ARRAY: + value = resultSet.getBytes(field.getEffectiveStorageName()); + break; + // fallthrough + case OracleDataTypesMapping.UUID: + case OracleDataTypesMapping.JSON: value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); break; default: From bc0f64060c91f6f15928a2361879c5225dac7a4f Mon Sep 17 00:00:00 2001 From: psilberk Date: Tue, 10 Jun 2025 23:07:44 -0700 Subject: [PATCH 14/64] Adding Book class and pom fixes --- data/semantickernel-data-postgres/pom.xml | 1 - .../semantickernel-learn-resources/pom.xml | 11 +++- .../data/vectorstores/oracle/Book.java | 45 ++++++++++++++++ .../data/vectorstores/oracle/Main.java | 51 ++++++++----------- 4 files changed, 75 insertions(+), 33 deletions(-) create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 4382122b..73591658 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -10,7 +10,6 @@ ../../pom.xml - com.microsoft.semantic-kernel semantickernel-data-postgres Semantic Kernel PostreSQL connector Provides a PostreSQL connector for the Semantic Kernel diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 68062bf4..b8a1ea0b 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -41,7 +41,16 @@ com.microsoft.semantic-kernel semantickernel-data-redis - + + com.microsoft.semantic-kernel + semantickernel-data-oracle + 1.4.4-RC2-SNAPSHOT + + + com.microsoft.semantic-kernel + semantickernel-data-postgres + 1.4.4-RC2-SNAPSHOT + org.apache.logging.log4j log4j-api diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java new file mode 100644 index 00000000..4812c778 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; + +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import java.util.List; + +public class Book { + + @VectorStoreRecordKey + private final String isbn; + + public Book(String isbn, String title, String author, int pages, + List tags, String summary, List summaryEmbedding) { + this.isbn = isbn; + this.title = title; + this.author = author; + this.pages = pages; + this.tags = tags; + this.summary = summary; + this.summaryEmbedding = summaryEmbedding; + } + + @VectorStoreRecordData(isFilterable = true) + private final String title; + + @VectorStoreRecordData(isFilterable = true) + private final String author; + + @VectorStoreRecordData + private final int pages; + + @VectorStoreRecordData(isFilterable = true) + private final List tags; + + @VectorStoreRecordData( isFilterable = true, isFullTextSearchable = true ) + private final String summary; + + @VectorStoreRecordVector(dimensions = 4, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) + private final List summaryEmbedding; + +} \ No newline at end of file diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java index fe37be8a..d71a0f7e 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -3,13 +3,17 @@ import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; -import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollection; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import oracle.jdbc.datasource.impl.OracleDataSource; public class Main { - public static void main(String[] args) { + public static void main(String[] args) throws SQLException { + // Configure the data source OracleDataSource dataSource = new OracleDataSource(); dataSource.setURL("jdbc:oracle:thin:@localhost:1521/FREEPDB1"); @@ -29,50 +33,35 @@ public static void main(String[] args) { .build()) .build(); - VectorStoreRecordCollection collection = vectorStore.getCollection( - "skhotels", - JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(Hotel.class) + VectorStoreRecordCollection collection = vectorStore.getCollection( + "books", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Book.class) .build()); // Create the collection if it doesn't exist yet. collection.createCollectionIfNotExistsAsync().block(); - collection.upsertBatchAsync(getHotels(), null).block(); + collection.upsertBatchAsync(books, null).block(); // Retrieve the upserted record. - var retrievedHotel = collection.getAsync("1", null).block(); + //var retrievedBook = collection.getAsync("1", null).block(); // Generate a vector for your search text, using your chosen embedding generation implementation. // Just showing a placeholder method here for brevity. // var searchVector = generateEmbeddingsAsync( - // "I'm looking for a hotel where customer happiness is the priority.").block(); + // "I'm looking for a Book where customer happiness is the priority.").block(); // Do the search. // var searchResult = collection.searchAsync(searchVector, VectorSearchOptions.builder() // .withTop(1).build()).block(); - // Hotel record = searchResult.getResults().get(0).getRecord(); - // System.out.printf("Found hotel description: %s\n", record.getDescription()); + // Book record = searchResult.getResults().get(0).getRecord(); + // System.out.printf("Found Book description: %s\n", record.getDescription()); } - private static List getHotels() { - return Arrays.asList( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", - Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, - 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", - Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, - 4.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", - Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, - 5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", - Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, - 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", - Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, - 4.0)); - } + static List books = Arrays.asList( + new Book("1", "one", "sking", 0, null, "sum", null)); + } \ No newline at end of file From 3cbee729a3e55a302f5d7cc00df54bd1e6ab5256 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Wed, 11 Jun 2025 11:45:30 +0200 Subject: [PATCH 15/64] Missing getter methods and wrong null check --- .../OracleVectorStoreQueryProvider.java | 2 +- .../data/vectorstores/oracle/Book.java | 34 +++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index b74226fe..71da3199 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -309,7 +309,7 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record if (field instanceof VectorStoreRecordVectorField) { // Convert the vector field to a string if (!field.getFieldType().equals(String.class)) { - double[] values = valueNode == null ? null : StreamSupport.stream(((ArrayNode)valueNode).spliterator(), false).mapToDouble(d -> d.asDouble()).toArray(); + double[] values = valueNode.isNull() ? null : StreamSupport.stream(((ArrayNode)valueNode).spliterator(), false).mapToDouble(d -> d.asDouble()).toArray(); statement.setObject(i + 1, values, OracleType.VECTOR_FLOAT64); System.out.println("Set values: " + values); continue; diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java index 4812c778..aeaff6e8 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java @@ -10,9 +10,6 @@ public class Book { - @VectorStoreRecordKey - private final String isbn; - public Book(String isbn, String title, String author, int pages, List tags, String summary, List summaryEmbedding) { this.isbn = isbn; @@ -24,6 +21,10 @@ public Book(String isbn, String title, String author, int pages, this.summaryEmbedding = summaryEmbedding; } + + @VectorStoreRecordKey + private final String isbn; + @VectorStoreRecordData(isFilterable = true) private final String title; @@ -42,4 +43,31 @@ public Book(String isbn, String title, String author, int pages, @VectorStoreRecordVector(dimensions = 4, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) private final List summaryEmbedding; + public String getIsbn() { + return isbn; + } + + public String getTitle() { + return title; + } + + public String getAuthor() { + return author; + } + + public int getPages() { + return pages; + } + + public List getTags() { + return tags; + } + + public String getSummary() { + return summary; + } + + public List getSummaryEmbedding() { + return summaryEmbedding; + } } \ No newline at end of file From c3db0367affd318079e94b9fe6c1b004f05173c2 Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 14 Jun 2025 00:50:03 -0700 Subject: [PATCH 16/64] add test for key type --- ...OracleVectorStoreRecordCollectionTest.java | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 4ec1599d..05c174a2 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -1,5 +1,7 @@ package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; @@ -16,6 +18,7 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; + import oracle.jdbc.OracleConnection; import oracle.jdbc.datasource.impl.OracleDataSource; import org.junit.jupiter.api.BeforeAll; @@ -33,7 +36,9 @@ import java.sql.Statement; import java.time.Duration; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -334,6 +339,69 @@ public void searchWithTagFilter() { assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); } + @ParameterizedTest + @MethodSource("supportedKeyTypes") + void testKeyTypes(String suffix, Class keyType, Object keyValue) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(keyType) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .build(); + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vec") + .withStorageName("vec") + .withFieldType(List.class) + .withDimensions(2) + .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_keytype_" + suffix; + + VectorStoreRecordCollection collectionRaw = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(DummyRecordForKeyTypes.class) + .withRecordDefinition(definition) + .build()); + + VectorStoreRecordCollection collection = + (VectorStoreRecordCollection) collectionRaw; + + collection.createCollectionAsync().block(); + + DummyRecordForKeyTypes record = new DummyRecordForKeyTypes(keyValue, "dummyValue", Arrays.asList(1.0f, 2.0f)); + collection.upsertAsync(record, null).block(); + + DummyRecordForKeyTypes result = collection.getAsync(keyValue, null).block(); + assertNotNull(result); + assertEquals("dummyValue", result.getDummy()); + + collection.deleteCollectionAsync().block(); + } + @Nested class HNSWIndexTests { @Test @@ -502,4 +570,48 @@ private static Stream parametersExactSearch() { Arguments.of (DistanceFunction.UNDEFINED, Arrays.asList(0.1000d, 18.9081d, 19.9669d)) ); } + + // commented out temporarily because only String type key is supported in + // JDBCVectorStoreRecordCollection#getKeyFromRecord: + // ... + // return (String) keyField.get(data); + // ... + // thus upsertAync/getAsync won't work + private static Stream supportedKeyTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123")/*, + Arguments.of("integer", Integer.class, 321), + Arguments.of("long", Long.class, 5L), + Arguments.of("short", Short.class, (short) 3), + Arguments.of("uuid", UUID.class, UUID.randomUUID())*/ + ); + } + + private static class DummyRecordForKeyTypes { + private final Object id; + private final String dummy; + private final List vec; + @JsonCreator + public DummyRecordForKeyTypes( + @JsonProperty("id")Object id, + @JsonProperty("dummy") String dummy, + @JsonProperty("vec") List vec) { + this.id = id; + this.dummy = dummy; + this.vec = vec; + } + + public Object getId() { + return id; + } + + public String getDummy() { + return dummy; + } + + @Override + public String toString() { + return String.valueOf(id); + } + } } From 2306e210f5f3645248313ced01283c663c5ed7a7 Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 14 Jun 2025 00:59:38 -0700 Subject: [PATCH 17/64] enable test for COSINE_DISTANCE, COSINE_SIMILARITY, DOT_PRODUCT in Hotel.java --- .../semantickernel/data/jdbc/oracle/Hotel.java | 6 +++--- .../OracleVectorStoreRecordCollectionTest.java | 15 ++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index 197cb8c3..667d771e 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -77,9 +77,9 @@ protected Hotel( this.tags = tags; this.description = description; this.euclidean = euclidean; - this.cosineDistance = euclidean; - this.cosineSimilarity = euclidean; - this.dotProduct = euclidean; + this.cosineDistance = cosineDistance; + this.cosineSimilarity = cosineSimilarity; + this.dotProduct = dotProduct; this.rating = rating; } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 05c174a2..e71a9655 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -141,21 +141,26 @@ public void clearCollection() { } private static List getHotels() { + List vec1 = Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f); + List vec2 = Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f); + List vec3 = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f); + List vec4 = Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f); + List vec5 =Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f); return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", - Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + vec1, vec1, vec1, vec1, 4.0), new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", - Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, + vec2, vec2, vec2, vec2, 4.0), new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", - Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, + vec3, vec3, vec3, vec3, 5.0), new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", - Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, + vec4, vec4, vec4, vec4, 4.0), new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", - Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + vec5, vec5, vec5, vec5, 4.0)); } From 0bf9199c280d88c254b5c46a24100133592cd088 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 17 Jun 2025 21:11:59 +0200 Subject: [PATCH 18/64] Refactoring --- .../jdbc/JDBCVectorStoreRecordCollection.java | 6 +- .../jdbc/oracle/OracleDataTypesMapping.java | 4 + .../oracle/OracleVectorStoreFieldHelper.java | 307 ++++++++++++++++++ .../OracleVectorStoreQueryProvider.java | 297 ++++++----------- .../data/jdbc/oracle/Hotel.java | 22 +- ...OracleVectorStoreRecordCollectionTest.java | 25 +- .../VectorStoreRecordDefinition.java | 2 +- .../VectorStoreRecordVectorField.java | 6 +- 8 files changed, 446 insertions(+), 223 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java index 2d75af14..81395589 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -34,10 +34,10 @@ public class JDBCVectorStoreRecordCollection implements SQLVectorStoreRecordCollection { private final String collectionName; - private final VectorStoreRecordDefinition recordDefinition; - private final VectorStoreRecordMapper vectorStoreRecordMapper; + protected final VectorStoreRecordDefinition recordDefinition; + protected final VectorStoreRecordMapper vectorStoreRecordMapper; private final JDBCVectorStoreRecordCollectionOptions options; - private final SQLVectorStoreQueryProvider queryProvider; + protected final SQLVectorStoreQueryProvider queryProvider; /** * Creates a new instance of the {@link JDBCVectorStoreRecordCollection}. diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index f441a440..4b74527b 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -1,5 +1,8 @@ package com.microsoft.semantickernel.data.jdbc.oracle; +/** + * Defines oracle database type constants for supported field types. + */ public class OracleDataTypesMapping { public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; public static final String STRING_CLOB = "CLOB"; @@ -15,4 +18,5 @@ public class OracleDataTypesMapping { public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; public static final String UUID = "RAW(16)"; public static final String JSON = "JSON"; + public static final String VECTOR_FLOAT = "VECTOR(%s, FLOAT32)"; } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java new file mode 100644 index 00000000..3ae09ed4 --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -0,0 +1,307 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import oracle.jdbc.OracleTypes; +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Helper class for field operations. + */ +public class OracleVectorStoreFieldHelper { + private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); + + /** + * Maps supported key java classes to Oracle database types + */ + private static final HashMap, String> supportedKeyTypes = new HashMap() { + { + put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); + put(short.class, OracleDataTypesMapping.SHORT); + put(Short.class, OracleDataTypesMapping.SHORT); + put(int.class, OracleDataTypesMapping.INTEGER); + put(Integer.class, OracleDataTypesMapping.INTEGER); + put(long.class, OracleDataTypesMapping.LONG); + put(Long.class, OracleDataTypesMapping.LONG); + put(UUID .class, OracleDataTypesMapping.UUID); + } + }; + + /** + * Maps supported vector java classes to Oracle database types + */ + private static final Map, String> supportedVectorTypes = new HashMap() { + { + put(String.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(List.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); + put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); +/* + put(byte[].class,"VECTOR(%s, INT8)"); + put(Byte[].class,"VECTOR(%s, INT8)"); + put(double[].class,"VECTOR(%s, FLOAT64)"); + put(Double[].class,"VECTOR(%s, FLOAT64)"); + put(boolean[].class,"VECTOR(%s, BINARY)"); + put(Boolean[].class,"VECTOR(%s, BINARY)"); + */ + } + }; + + /** + * Maps supported data java classes to Oracle database types + */ + private static final HashMap, String> supportedDataTypes = new HashMap() { + { + put(byte.class, OracleDataTypesMapping.BYTE); + put(Byte.class, OracleDataTypesMapping.BYTE); + put(short.class, OracleDataTypesMapping.SHORT); + put(Short.class, OracleDataTypesMapping.SHORT); + put(int.class, OracleDataTypesMapping.INTEGER); + put(Integer.class, OracleDataTypesMapping.INTEGER); + put(long.class, OracleDataTypesMapping.LONG); + put(Long.class, OracleDataTypesMapping.LONG); + put(Float.class, OracleDataTypesMapping.FLOAT); + put(float.class, OracleDataTypesMapping.FLOAT); + put(Double.class, OracleDataTypesMapping.DOUBLE); + put(double.class, OracleDataTypesMapping.DOUBLE); + put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); + put(Boolean.class, OracleDataTypesMapping.BOOLEAN); + put(boolean.class, OracleDataTypesMapping.BOOLEAN); + put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); + put(UUID.class, OracleDataTypesMapping.UUID); + put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); + put(List.class, OracleDataTypesMapping.JSON); + } + + }; + + /** + * Maps vector type to OracleTypes. Only needed if types other than FLOAT_32 are supported. + */ + private static final Map, Integer> mapOracleTypeToVector = new HashMap() { + { + put(float[].class, OracleTypes.VECTOR_FLOAT32); + put(Float[].class, OracleTypes.VECTOR_FLOAT32); +/* + put(byte[].class, OracleTypes.VECTOR_INT8); + put(Byte[].class, OracleTypes.VECTOR_INT8); + put(Double[].class, OracleTypes.VECTOR_FLOAT64); + put(double[].class, OracleTypes.VECTOR_FLOAT64); + put(Boolean[].class, OracleTypes.VECTOR_BINARY); + put(boolean[].class, OracleTypes.VECTOR_BINARY); +*/ + } + }; + + /** + * Gets the mapping between the supported Java key types and the Oracle database type. + * + * @return the mapping between the supported Java key types and the Oracle database type. + */ + public static HashMap, String> getSupportedKeyTypes() { + return supportedKeyTypes; + } + + /** + * Gets the mapping between the supported Java data types and the Oracle database type. + * + * @return the mapping between the supported Java data types and the Oracle database type. + */ + public static Map, String> getSupportedDataTypes( + StringTypeMapping stringTypeMapping, int defaultVarCharLength) { + + if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { + supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); + } else { + supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); + } + return supportedDataTypes; + } + + /** + * Gets the mapping between the supported Java data types and the Oracle database type. + * + * @return the mapping between the supported Java data types and the Oracle database type. + */ + public static Map, String> getSupportedVectorTypes() { + return supportedVectorTypes; + } + + /** + * Generates the statement to create the index according to the vector field definition. + * + * @return the CREATE VECTOR INDEX statement to create the index according to the vector + * field definition. + */ + public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField field, String collectionTableName) { + switch (field.getIndexKind()) { + case IVFFLAT: + return "CREATE VECTOR INDEX IF NOT EXISTS " + + getIndexName(field.getEffectiveStorageName()) + + " ON " + + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + " ORGANIZATION NEIGHBOR PARTITIONS " + + " WITH DISTANCE COSINE " + + "PARAMETERS ( TYPE IVF )"; + case HNSW: + return "CREATE VECTOR INDEX IF NOT EXISTS " + getIndexName(field.getEffectiveStorageName()) + + " ON " + + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + "ORGANIZATION INMEMORY GRAPH " + + "WITH DISTANCE COSINE " + + "PARAMETERS (TYPE HNSW)"; + case UNDEFINED: + return null; + default: + LOGGER.warning("Unsupported index kind: " + field.getIndexKind()); + return null; + } + } + + /** + * Generates the statement to create the index according to the field definition. + * + * @return the CREATE INDEX statement to create the index according to the field definition. + */ + public static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { + if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { + String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; + return String.format(dataFieldIndex, + collectionTableName + "_" + dataField.getEffectiveStorageName(), + collectionTableName, + dataField.getEffectiveStorageName(), + getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); + } else { + String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; + return String.format(dataFieldIndex, + collectionTableName + "_" + dataField.getEffectiveStorageName(), + collectionTableName, + dataField.getEffectiveStorageName() + ); + } + } + + /** + * Gets the function that allows to return the function that converts the JSON value to the + * data type. + * @param jdbcType The JDBC type. + * @return the function that allows to return the function that converts the JSON value to the + * data type. + */ + private static String getFunctionForType(String jdbcType) { + switch (jdbcType) { + case OracleDataTypesMapping.BOOLEAN: + return "boolean()"; + case OracleDataTypesMapping.BYTE: + case OracleDataTypesMapping.SHORT: + case OracleDataTypesMapping.INTEGER: + case OracleDataTypesMapping.LONG: + case OracleDataTypesMapping.FLOAT: + case OracleDataTypesMapping.DOUBLE: + case OracleDataTypesMapping.DECIMAL: + return "numberOnly()"; + case OracleDataTypesMapping.OFFSET_DATE_TIME: + return "timestamp()"; + default: + return "string()"; + } + } + + /** + * Gets the type of the vector given the field definition. This method is not needed if only + * + * @param field the vector field definition. + * @return returns the type of vector for the given field type. + */ + public static String getTypeForVectorField(VectorStoreRecordVectorField field) { + String dimension = field.getDimensions() > 0 ? String.valueOf(field.getDimensions()) : "*"; + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); +/* Not needed since all types are FLOAT32 + if (field.getFieldSubType() != null) { + String vectorType; + switch (field.getFieldSubType().getName()) { + case "java.lang.Double": + vectorType = "FLOAT64"; + break; + case "java.lang.Byte": + vectorType = "INT8"; + break; + case "java.lang.Boolean": + vectorType = "BINARY"; + break; + default: + vectorType = "FLOAT32"; + } + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension, vectorType); + } else { + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); + } + */ + } + + /** + * Gets the JDBC oracle of the vector field definition. + * @param field the vector field definition. + * @return the JDBC oracle type. + */ + public static int getOracleTypeForField(VectorStoreRecordVectorField field) { + if (field.getFieldSubType() == null) { + return mapOracleTypeToVector.get(field.getFieldType()).intValue(); + } else { + switch (field.getFieldSubType().getName()) { + case "java.lang.Double": + return OracleTypes.VECTOR_FLOAT64; + case "java.lang.Byte": + return OracleTypes.VECTOR_INT8; + case "java.lang.Boolean": + return OracleTypes.VECTOR_BINARY; + default: + return OracleTypes.VECTOR_FLOAT32; + } + } + } + + /** + * Generates the index name given the field name. by suffixing "_VECTOR_INDEX" to the field name. + * @param effectiveStorageName the field name. + * @return the index name. + */ + private static String getIndexName(String effectiveStorageName) { + return effectiveStorageName + "_VECTOR_INDEX"; + } + + /** + * Returns vector columns names and types for CREATE TABLE statement + * @param fields list of vector record fields. + * @return comma separated list of columns and types for CREATE TABLE statement. + */ + public static String getVectorColumnNamesAndTypes(List fields) { + List columns = fields.stream() + .map(field -> field.getEffectiveStorageName() + " " + + OracleVectorStoreFieldHelper.getTypeForVectorField(field) + ).collect(Collectors.toList()); + + return String.join(", ", columns); + } + + /** + * Returns key column names and type for key column for CREATE TABLE statement + * @param field the key field. + * @return column name and type of the key field for CREATE TABLE statement. + */ + public static String getKeyColumnNameAndType(VectorStoreRecordKeyField field) { + return field.getEffectiveStorageName() + " " + supportedKeyTypes.get(field.getFieldType()); + } + +} diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 12466a6f..b32e2a82 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -15,38 +15,33 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import oracle.jdbc.OracleResultSet; import oracle.jdbc.OracleStatement; -import oracle.jdbc.OracleType; import oracle.jdbc.OracleTypes; import javax.annotation.Nonnull; +import javax.annotation.concurrent.GuardedBy; import javax.sql.DataSource; -import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.Statement; import java.sql.SQLException; -import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +/** + * JDBC Vector Store for the Oracle Database + */ public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { // This could be removed if super.collectionTable made protected @@ -57,7 +52,7 @@ public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider private static final Object dbCreationLock = new Object(); - Logger logger = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); + private static final Logger logger = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); public enum StringTypeMapping { /** @@ -70,6 +65,15 @@ public enum StringTypeMapping { USE_VARCHAR } + /** + * Constructor + * @param dataSource the datasiyrce + * @param collectionsTable the collections table name + * @param prefixForCollectionTables the prefix for the collection table name + * @param defaultVarcharSize the size of VARCHAR columns + * @param stringTypeMapping the storage type of string columns (VARCHAR or CLOB) + * @param objectMapper the object mapper. + */ private OracleVectorStoreQueryProvider( @Nonnull DataSource dataSource, @Nonnull String collectionsTable, @@ -81,113 +85,16 @@ private OracleVectorStoreQueryProvider( dataSource, collectionsTable, prefixForCollectionTables, - buildSupportedKeyTypes(), - buildSupportedDataTypes(stringTypeMapping, defaultVarcharSize), - buildSupportedVectorTypes()); + OracleVectorStoreFieldHelper.getSupportedKeyTypes(), + OracleVectorStoreFieldHelper.getSupportedDataTypes(stringTypeMapping, defaultVarcharSize), + OracleVectorStoreFieldHelper.getSupportedVectorTypes()); this.collectionsTable = collectionsTable; this.objectMapper = objectMapper; } - private static HashMap, String> buildSupportedKeyTypes() { - HashMap, String> supportedKeyTypes = new HashMap<>(); - supportedKeyTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); - supportedKeyTypes.put(short.class, OracleDataTypesMapping.SHORT); - supportedKeyTypes.put(Short.class, OracleDataTypesMapping.SHORT); - supportedKeyTypes.put(int.class, OracleDataTypesMapping.INTEGER); - supportedKeyTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); - supportedKeyTypes.put(long.class, OracleDataTypesMapping.LONG); - supportedKeyTypes.put(Long.class, OracleDataTypesMapping.LONG); - supportedKeyTypes.put(UUID.class, OracleDataTypesMapping.UUID); - - return supportedKeyTypes; - } - private static Map, String> buildSupportedVectorTypes() { - HashMap, String> supportedVectorTypes = new HashMap<>(); - supportedVectorTypes.put(String.class, "VECTOR(%s)"); - supportedVectorTypes.put(List.class, "VECTOR(%s)"); - supportedVectorTypes.put(Collection.class, "VECTOR(%s)"); - return supportedVectorTypes; - } - - private static Map, String> buildSupportedDataTypes(StringTypeMapping stringTypeMapping, int defaultVarCharLength) { - HashMap, String> supportedDataTypes = new HashMap<>(); - if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { - supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); - } else { - supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); - } - supportedDataTypes.put(byte.class, OracleDataTypesMapping.BYTE); - supportedDataTypes.put(Byte.class, OracleDataTypesMapping.BYTE); - supportedDataTypes.put(short.class, OracleDataTypesMapping.SHORT); - supportedDataTypes.put(Short.class, OracleDataTypesMapping.SHORT); - supportedDataTypes.put(int.class, OracleDataTypesMapping.INTEGER); - supportedDataTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); - supportedDataTypes.put(long.class, OracleDataTypesMapping.LONG); - supportedDataTypes.put(Long.class, OracleDataTypesMapping.LONG); - supportedDataTypes.put(Float.class, OracleDataTypesMapping.FLOAT); - supportedDataTypes.put(float.class, OracleDataTypesMapping.FLOAT); - supportedDataTypes.put(Double.class, OracleDataTypesMapping.DOUBLE); - supportedDataTypes.put(double.class, OracleDataTypesMapping.DOUBLE); - supportedDataTypes.put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); - supportedDataTypes.put(Boolean.class, OracleDataTypesMapping.BOOLEAN); - supportedDataTypes.put(boolean.class, OracleDataTypesMapping.BOOLEAN); - supportedDataTypes.put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); - supportedDataTypes.put(UUID.class, OracleDataTypesMapping.UUID); - supportedDataTypes.put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); - supportedDataTypes.put(List.class, OracleDataTypesMapping.JSON); - return supportedDataTypes; - } - - private String createIndexForVectorField(String collectionName, VectorStoreRecordVectorField vectorField) { - switch (vectorField.getIndexKind()) { - case IVFFLAT: - return "CREATE VECTOR INDEX IF NOT EXISTS " - + getIndexName(vectorField.getEffectiveStorageName()) - + " ON " - + getCollectionTableName(collectionName) + "( " + vectorField.getEffectiveStorageName() + " ) " - + " ORGANIZATION NEIGHBOR PARTITIONS " - + " WITH DISTANCE COSINE " - + "PARAMETERS ( TYPE IVF )"; - case HNSW: - return "CREATE VECTOR INDEX IF NOT EXISTS " + getIndexName(vectorField.getEffectiveStorageName()) - + " ON " - + getCollectionTableName(collectionName) + "( " + vectorField.getEffectiveStorageName() + " ) " - + "ORGANIZATION INMEMORY GRAPH " - + "WITH DISTANCE COSINE " - + "PARAMETERS (TYPE HNSW)"; - case UNDEFINED: - return null; - default: - logger.warning("Unsupported index kind: " + vectorField.getIndexKind()); - return null; - } - } - - private String getIndexName(String effectiveStorageName) { - return effectiveStorageName + "_VECTOR_INDEX"; - } - - - protected String getVectorColumnNamesAndTypes(List fields, - Map, String> types) { - List columns = fields.stream() - .map(field -> validateSQLidentifier(field.getEffectiveStorageName()) + " " - + String.format(types.get(field.getFieldType()), field.getDimensions() > 0 ? field.getDimensions() + ", FLOAT32" : "FLOAT32")) - .collect(Collectors.toList()); - - return String.join(", ", columns); - } - - @Override - protected String getInsertCollectionQuery(String collectionsTable) { - return formatQuery( - "MERGE INTO %s existing "+ - "USING (SELECT ? AS collectionId FROM DUAL) new ON (existing.collectionId = new.collectionId) " + - "WHEN NOT MATCHED THEN INSERT (existing.collectionId) VALUES (new.collectionId)", - collectionsTable); - } - @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") + @GuardedBy("dbCreationLock") public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { @@ -199,11 +106,11 @@ public void createCollection(String collectionName, + "%s, " + "%s)", getCollectionTableName(collectionName), - getKeyColumnNameAndType(recordDefinition.getKeyField(), getSupportedDataTypes()), + OracleVectorStoreFieldHelper.getKeyColumnNameAndType(recordDefinition.getKeyField()), getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), getSupportedDataTypes()), - getVectorColumnNamesAndTypes(new ArrayList<>(vectorFields), - getSupportedVectorTypes())); + OracleVectorStoreFieldHelper.getVectorColumnNamesAndTypes( + new ArrayList<>(vectorFields))); String insertCollectionQuery = this.getInsertCollectionQuery(collectionsTable); @@ -213,28 +120,28 @@ public void createCollection(String collectionName, try (Statement statement = connection.createStatement()) { // Create table System.out.println(createStorageTable); - statement.addBatch(createStorageTable); + statement.execute(createStorageTable); // Index filterable columns for (VectorStoreRecordDataField dataField : recordDefinition.getDataFields()) { if (dataField.isFilterable()) { - String dataFieldIndex = createIndexForDataField(collectionName, dataField); + String dataFieldIndex = OracleVectorStoreFieldHelper.createIndexForDataField( + getCollectionTableName(collectionName), dataField, supportedDataTypes); System.out.println(dataFieldIndex); - statement.addBatch(dataFieldIndex); + statement.execute(dataFieldIndex); } } // Create indexed for vectorFields for (VectorStoreRecordVectorField vectorField : vectorFields) { - String createVectorIndex = createIndexForVectorField(collectionName, - vectorField); - + String createVectorIndex = OracleVectorStoreFieldHelper.getCreateVectorIndexStatement( + vectorField, getCollectionTableName(collectionName)); if (createVectorIndex != null) { System.out.println(createVectorIndex); - statement.addBatch(createVectorIndex); + statement.execute(createVectorIndex); } } - statement.executeBatch(); + //statement.executeBatch(); try (PreparedStatement insert = connection.prepareStatement( insertCollectionQuery)) { @@ -254,60 +161,48 @@ public void createCollection(String collectionName, } } - private String getKeyColumnNameAndType(VectorStoreRecordKeyField field, Map, String> types) { - return validateSQLidentifier(field.getEffectiveStorageName()) + " " - + types.get(field.getFieldType()); + @Override + protected String getInsertCollectionQuery(String collectionsTable) { + return formatQuery( + "MERGE INTO %s existing "+ + "USING (SELECT ? AS collectionId FROM DUAL) new ON (existing.collectionId = new.collectionId) " + + "WHEN NOT MATCHED THEN INSERT (existing.collectionId) VALUES (new.collectionId)", + collectionsTable); } - private String createIndexForDataField(String collectionName, VectorStoreRecordDataField dataField) { - if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { - String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; - return formatQuery(dataFieldIndex, - getCollectionTableName(collectionName) + "_" + dataField.getEffectiveStorageName(), - getCollectionTableName(collectionName), - dataField.getEffectiveStorageName(), - getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); - } else { - String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; - return formatQuery(dataFieldIndex, - getCollectionTableName(collectionName) + "_" + dataField.getEffectiveStorageName(), - getCollectionTableName(collectionName), - dataField.getEffectiveStorageName() - ); - } + @Override + public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { - } + final String NEW_VALUE = "new"; + final String EXISTING_VALUE = "existing"; - private String getFunctionForType(String jdbcType) { - switch (jdbcType) { - case "BOOLEAN": - return "boolean()"; - case "INTEGER": - case "LONG": - case "REAL": - case "DOUBLE PRECISION": - return "numberOnly()"; - case "TIMESTAMPTZ": - return "timestamp()"; - default: - return "string()"; - } - } + String insertNewFieldList = recordDefinition.getAllFields().stream() + .map(f -> NEW_VALUE + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); - @Override - public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + String insertExistingFieldList = recordDefinition.getAllFields().stream() + .map(f -> EXISTING_VALUE + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + + String updateFieldList = recordDefinition.getAllFields().stream() + .filter(f -> f != recordDefinition.getKeyField()) + .map(f -> EXISTING_VALUE + "." + f.getEffectiveStorageName() + " = " + NEW_VALUE + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + + String namedWildcard = recordDefinition.getAllFields().stream().map(f -> "? " + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); String upsertQuery = formatQuery("MERGE INTO %s existing "+ "USING (SELECT %s FROM DUAL) new ON (existing.%s = new.%s) " + "WHEN MATCHED THEN UPDATE SET %s " + "WHEN NOT MATCHED THEN INSERT (%s) VALUES (%s)", getCollectionTableName(collectionName), - getNamedWildcard(recordDefinition.getAllFields()), + namedWildcard, getKeyColumnName(recordDefinition.getKeyField()), getKeyColumnName(recordDefinition.getKeyField()), - getUpdateFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "existing", "new"), - getInsertFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "existing"), - getInsertFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "new")); + updateFieldList, + insertExistingFieldList, + insertNewFieldList); System.out.println(upsertQuery); try (Connection connection = dataSource.getConnection(); @@ -335,8 +230,13 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record if (field instanceof VectorStoreRecordVectorField) { // Convert the vector field to a string if (!field.getFieldType().equals(String.class)) { - double[] values = valueNode == null ? null : StreamSupport.stream(((ArrayNode)valueNode).spliterator(), false).mapToDouble(d -> d.asDouble()).toArray(); - statement.setObject(i + 1, values, OracleType.VECTOR_FLOAT64); + double[] values = valueNode.isNull() + ? null + : StreamSupport.stream(( + (ArrayNode)valueNode).spliterator(), false) + .mapToDouble(d -> d.asDouble()).toArray(); + statement.setObject(i + 1, values, + OracleVectorStoreFieldHelper.getOracleTypeForField((VectorStoreRecordVectorField)field)); System.out.println("Set values: " + values); continue; } @@ -357,23 +257,6 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record } } } - private String getInsertFieldList(VectorStoreRecordKeyField key, List fields, String alias) { - return fields.stream().map(f -> alias + "." + f.getEffectiveStorageName()) - .collect(Collectors.joining(", ")); - } - - private String getUpdateFieldList(VectorStoreRecordKeyField key, List fields, String oldAlias, String newAlias) { - return fields.stream().filter(f -> f != key).map(f -> oldAlias + "." + f.getEffectiveStorageName() + " = " + - newAlias + "." + f.getEffectiveStorageName()) - .collect(Collectors.joining(", ")); - - } - - - private String getNamedWildcard(List fields) { - return fields.stream().map(f -> "? " + f.getEffectiveStorageName()) - .collect(Collectors.joining(", ")); - } @Override public VectorSearchResults search(String collectionName, List vector, @@ -430,10 +313,12 @@ public VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List(records); } + private void defineDataColumnType(int columnIndex, OracleStatement statement, Class fieldType) throws SQLException { // swich between supported classes and define the column type on the statement switch (supportedDataTypes.get(fieldType)) { - case "CLOB": + case OracleDataTypesMapping.STRING_CLOB: statement.defineColumnType(columnIndex, OracleTypes.CLOB, Integer.MAX_VALUE); break; - case "INTEGER": + case OracleDataTypesMapping.BYTE: + statement.defineColumnType(columnIndex, OracleTypes.NUMBER); + break; + case OracleDataTypesMapping.SHORT: + statement.defineColumnType(columnIndex, OracleTypes.NUMBER); + break; + case OracleDataTypesMapping.INTEGER: statement.defineColumnType(columnIndex, OracleTypes.INTEGER); break; - case "LONG": + case OracleDataTypesMapping.LONG: statement.defineColumnType(columnIndex, OracleTypes.BIGINT); break; - case "REAL": - statement.defineColumnType(columnIndex, OracleTypes.REAL); + case OracleDataTypesMapping.FLOAT: + statement.defineColumnType(columnIndex, OracleTypes.BINARY_FLOAT); break; - case "DOUBLE PRECISION": + case OracleDataTypesMapping.DOUBLE: statement.defineColumnType(columnIndex, OracleTypes.BINARY_DOUBLE); break; - case "BOOLEAN": + case OracleDataTypesMapping.DECIMAL: + statement.defineColumnType(columnIndex, OracleTypes.BINARY_DOUBLE); + break; + case OracleDataTypesMapping.BOOLEAN: statement.defineColumnType(columnIndex, OracleTypes.BOOLEAN); break; - case "TIMESTAMPTZ": + case OracleDataTypesMapping.OFFSET_DATE_TIME: statement.defineColumnType(columnIndex, OracleTypes.TIMESTAMPTZ); break; - case "JSON": + case OracleDataTypesMapping.JSON: statement.defineColumnType(columnIndex, OracleTypes.JSON, Integer.MAX_VALUE); break; + case OracleDataTypesMapping.UUID: + case OracleDataTypesMapping.BYTE_ARRAY: + statement.defineColumnType(columnIndex, OracleTypes.RAW); default: statement.defineColumnType(columnIndex, OracleTypes.VARCHAR); } @@ -542,11 +438,6 @@ public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { return String.format("JSON_EXISTS(%s, '$[*]?(@ == $v_%s)' PASSING ? AS \"v_%s\")", fieldName, fieldName, fieldName); } - - - public static Builder builder() { - return new Builder(); - } @Override public VectorStoreRecordMapper getVectorStoreRecordMapper( @@ -559,6 +450,10 @@ public VectorStoreRecordMapper getVectorStoreRecordM .build(); } + public static Builder builder() { + return new Builder(); + } + public static class Builder extends JDBCVectorStoreQueryProvider.Builder { diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index 197cb8c3..d1c848a6 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -40,15 +40,15 @@ public class Hotel { @JsonProperty("summaryEmbedding2") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) - private final List cosineDistance; + private final float[] cosineDistance; @JsonProperty("summaryEmbedding3") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY, indexKind = IndexKind.IVFFLAT) - private final List cosineSimilarity; + private final float[] cosineSimilarity; @JsonProperty("summaryEmbedding4") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT, indexKind = IndexKind.IVFFLAT) - private final List dotProduct; + private final Float[] dotProduct; @VectorStoreRecordData private double rating; @@ -66,9 +66,9 @@ protected Hotel( @JsonProperty("tags") List tags, @JsonProperty("summary") String description, @JsonProperty("summaryEmbedding1") List euclidean, - @JsonProperty("summaryEmbedding2") List cosineDistance, - @JsonProperty("summaryEmbedding3") List cosineSimilarity, - @JsonProperty("summaryEmbedding4") List dotProduct, + @JsonProperty("summaryEmbedding2") float[] cosineDistance, + @JsonProperty("summaryEmbedding3") float[] cosineSimilarity, + @JsonProperty("summaryEmbedding4") Float[] dotProduct, @JsonProperty("rating") double rating) { this.id = id; this.name = name; @@ -77,9 +77,9 @@ protected Hotel( this.tags = tags; this.description = description; this.euclidean = euclidean; - this.cosineDistance = euclidean; - this.cosineSimilarity = euclidean; - this.dotProduct = euclidean; + this.cosineDistance = cosineDistance; + this.cosineSimilarity = cosineSimilarity; + this.dotProduct = dotProduct; this.rating = rating; } @@ -107,11 +107,11 @@ public List getEuclidean() { return euclidean; } - public List getCosineDistance() { + public float[] getCosineDistance() { return cosineDistance; } - public List getDotProduct() { + public Float[] getDotProduct() { return dotProduct; } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 72ed3524..22d0259f 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -128,19 +128,34 @@ public void clearCollection() { private static List getHotels() { return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", - Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), + new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, + new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, + new Float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, 4.0), new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", - Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, + Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), + new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, + new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, + new Float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, 4.0), new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", - Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, + Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), + new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, + new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, + new Float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, 5.0), new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", - Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, + Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), + new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, + new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, + new Float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, 4.0), new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", - Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), + new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, + new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, + new Float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, 4.0)); } diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 8f8d6212..e769bb6f 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -210,7 +210,7 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) vectorFields.add(VectorStoreRecordVectorField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType()) + .withFieldType(field.getType(), List.class.equals(field.getType()) ? (Class)((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0] : null) .withDimensions(vectorAttribute.dimensions()) .withIndexKind(vectorAttribute.indexKind()) .withDistanceFunction(vectorAttribute.distanceFunction()) diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java index 00b7627a..b708d2fb 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java @@ -34,10 +34,11 @@ public VectorStoreRecordVectorField( @Nonnull String name, @Nullable String storageName, @Nonnull Class fieldType, + Class fieldSubType, int dimensions, @Nullable IndexKind indexKind, @Nullable DistanceFunction distanceFunction) { - super(name, storageName, fieldType); + super(name, storageName, fieldType, fieldSubType); this.dimensions = dimensions; this.indexKind = indexKind == null ? IndexKind.UNDEFINED : indexKind; this.distanceFunction = distanceFunction == null ? DistanceFunction.UNDEFINED @@ -130,7 +131,8 @@ public VectorStoreRecordVectorField build() { throw new IllegalArgumentException("dimensions must be greater than 0"); } - return new VectorStoreRecordVectorField(name, storageName, fieldType, dimensions, + return new VectorStoreRecordVectorField(name, storageName, fieldType, fieldSubType, + dimensions, indexKind, distanceFunction); } From 434b9e66918db86f0f2333aa140a6ec612a05d2d Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Tue, 17 Jun 2025 23:17:44 -0700 Subject: [PATCH 19/64] Add test for supported data type(timestamp and uuid not included) --- ...OracleVectorStoreRecordCollectionTest.java | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index e71a9655..d1505f7e 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -29,20 +29,25 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.time.Duration; +import java.time.OffsetDateTime; import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -407,6 +412,87 @@ void testKeyTypes(String suffix, Class keyType, Object keyValue) { collection.deleteCollectionAsync().block(); } + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFieldValue, Class fieldSubType) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dataField; + if (fieldSubType != null) { + dataField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(dataFieldType, fieldSubType) + .isFilterable(true) + .build(); + } else { + dataField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(dataFieldType) + .isFilterable(true) + .build(); + } + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vec") + .withStorageName("vec") + .withFieldType(List.class) + .withDimensions(2) + .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dataField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_datatype_" + dataFieldName; + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions. builder() + .withRecordClass(DummyRecordForDataTypes.class) + .withRecordDefinition(definition).build()); + + collection.createCollectionAsync().block(); + + String key = "testid"; + + DummyRecordForDataTypes record = + new DummyRecordForDataTypes(key, dataFieldValue, Arrays.asList(1.0f, 2.0f)); + + collection.upsertAsync(record, null).block(); + + DummyRecordForDataTypes result = collection.getAsync(key, null).block(); + assertNotNull(result); + + if (dataFieldValue instanceof Number && result.getDummy() instanceof Number) { + assertEquals(((Number) dataFieldValue).doubleValue(), ((Number) result.getDummy()).doubleValue()); + } else if (dataFieldValue instanceof byte[]) { + assertArrayEquals((byte[]) dataFieldValue, (byte[]) result.getDummy()); + } else { + assertEquals(dataFieldValue, result.getDummy()); + } + + collection.deleteCollectionAsync().block(); + } + @Nested class HNSWIndexTests { @Test @@ -592,6 +678,25 @@ private static Stream supportedKeyTypes() { ); } + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123", null), + Arguments.of("boolean_true", Boolean.class, true, null), + Arguments.of("boolean_false", Boolean.class, false, null), + Arguments.of("byte", Byte.class, (byte) 127, null), + Arguments.of("short", Short.class, (short) 3, null), + Arguments.of("integer", Integer.class, 321, null), + Arguments.of("long", Long.class, 5L, null), + Arguments.of("float", Float.class, 3.14f, null), + Arguments.of("double", double.class, 3.14159265358d, null), + Arguments.of("decimal", BigDecimal.class, new BigDecimal("12345.67"), null), + //Arguments.of("timestamp", OffsetDateTime.class, OffsetDateTime.now(), null) + //Arguments.of("uuid", UUID.class, UUID.randomUUID(), null) + Arguments.of("byte_array", byte[].class, "abc".getBytes(StandardCharsets.UTF_8), null), + Arguments.of("json", List.class, Arrays.asList("a", "s", "d"), String.class) + ); + } + private static class DummyRecordForKeyTypes { private final Object id; private final String dummy; @@ -619,4 +724,32 @@ public String toString() { return String.valueOf(id); } } + + private static class DummyRecordForDataTypes { + private final String id; + private final Object dummy; + private final List vec; + @JsonCreator + public DummyRecordForDataTypes( + @JsonProperty("id") String id, + @JsonProperty("dummy") Object dummy, + @JsonProperty("vec") List vec) { + this.id = id; + this.dummy = dummy; + this.vec = vec; + } + + public String getId() { + return id; + } + + public Object getDummy() { + return dummy; + } + + @Override + public String toString() { + return String.valueOf(id); + } + } } From 1c83d62a1f8a5da0dad0eca235ecb53f7b1ea6a3 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Fri, 20 Jun 2025 18:46:42 +0200 Subject: [PATCH 20/64] Test and bug fixes --- .../OracleVectorStoreQueryProvider.java | 83 ++++- .../OracleVectorStoreDataTypeSearchTest.java | 297 ++++++++++++++++++ .../oracle/OracleVectorStoreDataTypeTest.java | 1 + 3 files changed, 375 insertions(+), 6 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 3c3e4fcc..f7a32fd5 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -34,6 +34,7 @@ import javax.annotation.Nonnull; import javax.annotation.concurrent.GuardedBy; import javax.sql.DataSource; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.sql.Connection; import java.sql.PreparedStatement; @@ -331,7 +332,8 @@ public VectorSearchResults search(String collectionName, List parameters = getFilterParameters(options.getVectorSearchFilter()); String selectQuery = "SELECT " - + formatQuery("VECTOR_DISTANCE(%s, ?, %s) distance, ", vectorField.getEffectiveStorageName(), toOracleDistanceFunction(distanceFunction)) + + (vector == null ? "0 as distance, " : + formatQuery("VECTOR_DISTANCE(%s, ?, %s) distance, ", vectorField.getEffectiveStorageName(), toOracleDistanceFunction(distanceFunction))) + getQueryColumnsFromFields(fields) + " FROM " + getCollectionTableName(collectionName) + (filter != null && !filter.isEmpty() ? " WHERE " + filter : "") @@ -345,12 +347,16 @@ public VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List(records); } + private void setSearchParameter(PreparedStatement statement, int index, Class type, Object value) { + + try { + if (List.class.equals(type)) { + statement.setObject(index, objectMapper.writeValueAsString(value)); + System.out.println( + "Set values: " + objectMapper.writeValueAsString(value)); + return; + } + if (UUID.class.equals(type)) { + if (value == null) { + statement.setNull(index, OracleTypes.RAW); + } else { + UUID uuid = (UUID)value; + ByteBuffer bb = ByteBuffer.allocate(16); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + statement.setBytes(index, bb.array()); + System.out.println("Set values: " + uuid); + } + return; + } + if (OffsetDateTime.class.equals(type)) { + if (value == null) { + statement.setNull(index, OracleTypes.TIMESTAMPTZ); + } else { + OffsetDateTime offsetDateTime = (OffsetDateTime) value; + ((OraclePreparedStatement) statement).setTIMESTAMPTZ(index, + TIMESTAMPTZ.of(offsetDateTime)); + System.out.println("Set values: " + offsetDateTime); + } + return; + } + if (BigDecimal.class.equals(type)) { + if (value == null) { + statement.setNull(index, OracleTypes.DECIMAL); + } else { + BigDecimal bigDecimal = (BigDecimal) value; + ((OraclePreparedStatement) statement).setBigDecimal(index, + bigDecimal); + System.out.println("Set values: " + bigDecimal); + } + return; + } + System.out.println("Set parameter " + index + " to: " + value); + statement.setObject(index, value); + + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + private void defineDataColumnType(int columnIndex, OracleStatement statement, Class fieldType) throws SQLException { // swich between supported classes and define the column type on the statement @@ -481,6 +539,19 @@ public List getFilterParameters(VectorSearchFilter filter) { }).collect(Collectors.toList()); } + @Override + public String getEqualToFilter(EqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + Object value = filterClause.getValue(); + + if (value == null) { + return String.format("%s is NULL", fieldName); + } else { + return String.format("%s = ?", fieldName); + } + } + @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java new file mode 100644 index 00000000..4ce739a8 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -0,0 +1,297 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OracleVectorStoreDataTypeSearchTest extends OracleCommonVectorStoreRecordCollectionTest { + private static final double MIN_NUMBER = 1.0E-130; + private static final BigDecimal BIG_NUMBER = BigDecimal.valueOf(9999999999999999.99); + + + + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypesSearch (ClassWithAllBoxedTypes record) { + VectorStoreRecordCollection collection = setupBoxed(); + + collection.upsertAsync(record, null).block(); + + // boolean + VectorSearchResults results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("booleanValue", record.getBooleanValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getBooleanValue(), results.getResults().get(0).getRecord().getBooleanValue()); + + // byte + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("byteValue", record.getByteValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getByteValue(), results.getResults().get(0).getRecord().getByteValue()); + + // short + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("shortValue", record.getShortValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getShortValue(), results.getResults().get(0).getRecord().getShortValue()); + + // integer + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("integerValue", record.getIntegerValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getIntegerValue(), results.getResults().get(0).getRecord().getIntegerValue()); + + // long + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("longValue", record.getLongValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getLongValue(), results.getResults().get(0).getRecord().getLongValue()); + + // float + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("floatValue", record.getFloatValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getFloatValue(), results.getResults().get(0).getRecord().getFloatValue()); + + // double + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("doubleValue", record.getDoubleValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); + + // decimal + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("decimalValue", record.getDecimalValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + if (record.getDecimalValue() != null) { + assertEquals(0, record.getDecimalValue() + .compareTo(results.getResults().get(0).getRecord().getDecimalValue())); + } else { + assertEquals(record.getDecimalValue(), + results.getResults().get(0).getRecord().getDecimalValue()); + } + + // offset date time + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("offsetDateTimeValue", record.getOffsetDateTimeValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + if (record.getOffsetDateTimeValue() != null) { + assertTrue(record.getOffsetDateTimeValue() + .isEqual(results.getResults().get(0).getRecord().getOffsetDateTimeValue())); + } else { + assertEquals(record.getOffsetDateTimeValue(), + results.getResults().get(0).getRecord().getOffsetDateTimeValue()); + } + + // UUID + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("uuidValue", record.getUuidValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getUuidValue(), results.getResults().get(0).getRecord().getUuidValue()); + + // byte array + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("byteArrayValue", record.getByteArrayValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertArrayEquals(record.getByteArrayValue(), results.getResults().get(0).getRecord().getByteArrayValue()); + + collection.deleteCollectionAsync().block(); + + } + + + public VectorStoreRecordCollection setupBoxed() { + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = + vectorStore.getCollection("BoxedTypes", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(ClassWithAllBoxedTypes.class) + .build()).createCollectionAsync().block(); + + collection.createCollectionAsync().block(); + + return collection; + } + + + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllBoxedTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, + Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + OffsetDateTime.now(), UUID.randomUUID(), new byte[] {Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", null, null, null, null, null, + null, null, null, + null, null, null, + null, + null + ) + ) + ); + } + + private static Stream supportedDataPrimitiveTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, + Long.MIN_VALUE, + Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + OffsetDateTime.now(), UUID.randomUUID(), + new byte[]{Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, + Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), + BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, (byte) 0, (short) 0, 0, 0l, + 0f, 0d, null, + null, null, null, + null, + null + ) + ) + ); + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java index 07002b23..5c575950 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java @@ -123,6 +123,7 @@ void testPrimitiveDataTypes(ClassWithAllPrimitiveTypes values) { collection.deleteCollectionAsync().block(); } + private static Stream supportedDataTypes() { return Stream.of( Arguments.of( From a6fac68e531fbeb9fa9ac136c22badfa1c222fee Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Sat, 21 Jun 2025 00:20:02 +0200 Subject: [PATCH 21/64] BigDecimal mapping and value range --- .../data/jdbc/oracle/OracleDataTypesMapping.java | 2 +- .../OracleVectorStoreDataTypeSearchTest.java | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index 4b74527b..0258cb5f 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -14,7 +14,7 @@ public class OracleDataTypesMapping { public static final String LONG = "NUMBER(19)"; public static final String FLOAT = "BINARY_FLOAT"; public static final String DOUBLE = "BINARY_DOUBLE"; - public static final String DECIMAL = "NUMBER(18,2)"; + public static final String DECIMAL = "NUMBER"; public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; public static final String UUID = "RAW(16)"; public static final String JSON = "JSON"; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java index 4ce739a8..d99234ab 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -4,11 +4,9 @@ import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -24,7 +22,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class OracleVectorStoreDataTypeSearchTest extends OracleCommonVectorStoreRecordCollectionTest { - private static final double MIN_NUMBER = 1.0E-130; + private static final double MIN_DOUBLE = 1.0E-130; + private static final double MIN_DECIMAL = -1.0E125; private static final BigDecimal BIG_NUMBER = BigDecimal.valueOf(9999999999999999.99); @@ -120,6 +119,11 @@ void testDataTypesSearch (ClassWithAllBoxedTypes record) { assertEquals(1, results.getTotalCount()); assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); + System.out.println(record.getDecimalValue()); + System.out.println(record.getDecimalValue().doubleValue()); + System.out.println(results.getResults().get(0).getRecord().getDecimalValue()); + System.out.println(results.getResults().get(0).getRecord().getDecimalValue().doubleValue()); + // decimal results = collection.searchAsync( null, @@ -223,7 +227,7 @@ private static Stream supportedDataTypes() { Arguments.of( new ClassWithAllBoxedTypes( "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, - Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + Float.MIN_VALUE, MIN_DOUBLE, BigDecimal.valueOf(MIN_DECIMAL), OffsetDateTime.now(), UUID.randomUUID(), new byte[] {Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } @@ -265,7 +269,7 @@ private static Stream supportedDataPrimitiveTypes() { new ClassWithAllPrimitiveTypes( "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, - Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + Float.MIN_VALUE, MIN_DOUBLE, BigDecimal.valueOf(MIN_DECIMAL), OffsetDateTime.now(), UUID.randomUUID(), new byte[]{Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), From 2f421a84bca70b39ba8d00b40a06c154cc63b8e5 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 3 Jun 2025 17:01:00 +0200 Subject: [PATCH 22/64] Any tag filter --- .../OracleVectorStoreQueryProvider.java | 47 +++++++++++++++++-- ...OracleVectorStoreRecordCollectionTest.java | 34 ++++++++++++-- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 71da3199..f9ba7414 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -4,7 +4,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.jdbc.*; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; @@ -34,6 +37,7 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -387,13 +391,12 @@ public VectorSearchResults search(String collectionName, List getFilterParameters(VectorSearchFilter filter) { + if (filter == null + || filter.getFilterClauses().isEmpty()) { + return Collections.emptyList(); + } + + return filter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return equalToFilterClause.getValue(); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + return anyTagEqualToFilterClause.getValue(); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.toList()); + } + + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + + return String.format("JSON_EXISTS(%s, '$[*]?(@ == $v_%s)' PASSING ? AS \"v_%s\")", + fieldName, fieldName, fieldName); + } + + public static Builder builder() { return new Builder(); } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 546e251f..72ed3524 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -127,19 +127,19 @@ public void clearCollection() { private static List getHotels() { return Arrays.asList( - new Hotel("id_1", "Hotel 1", 1, 1.49d, null, "Hotel 1 description", + new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, 4.0), - new Hotel("id_2", "Hotel 2", 2, 1.44d, null, "Hotel 2 description with free-text search", + new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, 4.0), - new Hotel("id_3", "Hotel 3", 3, 1.53d, null, "Hotel 3 description", + new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, 5.0), - new Hotel("id_4", "Hotel 4", 4, 1.35d, null, "Hotel 4 description", + new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, 4.0), - new Hotel("id_5", "Hotel 5", 5, 1.89d, null,"Hotel 5 description", + new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, 4.0)); } @@ -300,6 +300,30 @@ public void searchWithFilter(DistanceFunction distanceFunction, double expectedD assertEquals(results.get(0).getScore(), expectedDistance, 0.0001d); } + + @Test + public void searchWithTagFilter() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() +// .withVectorFieldName("") + .withTop(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .anyTagEqualTo("tags", "three") + .build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(1, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); + } + private static Stream distanceFunctionAndDistance() { return Stream.of( Arguments.of (DistanceFunction.COSINE_DISTANCE, 0.8548d), From cf99cf7e77bee1a51828afe1437a1158c32a0fe2 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 10 Jun 2025 17:55:16 +0200 Subject: [PATCH 23/64] Added constants for type mapping and matched ODP type mapping when possible. --- .../jdbc/oracle/OracleDataTypesMapping.java | 18 ++++++ .../OracleVectorStoreQueryProvider.java | 64 +++++++++++++------ .../oracle/OracleVectorStoreRecordMapper.java | 32 +++++++--- 3 files changed, 84 insertions(+), 30 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java new file mode 100644 index 00000000..f441a440 --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -0,0 +1,18 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +public class OracleDataTypesMapping { + public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; + public static final String STRING_CLOB = "CLOB"; + public static final String BOOLEAN = "BOOLEAN"; + public static final String BYTE = "NUMBER(3)"; + public static final String BYTE_ARRAY = "RAW(2000)"; + public static final String SHORT = "NUMBER(5)"; + public static final String INTEGER = "NUMBER(10)"; + public static final String LONG = "NUMBER(19)"; + public static final String FLOAT = "BINARY_FLOAT"; + public static final String DOUBLE = "BINARY_DOUBLE"; + public static final String DECIMAL = "NUMBER(18,2)"; + public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; + public static final String UUID = "RAW(16)"; + public static final String JSON = "JSON"; +} diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index f9ba7414..fe51f7b2 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -29,6 +29,7 @@ import javax.annotation.Nonnull; import javax.sql.DataSource; +import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -41,6 +42,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -81,17 +83,25 @@ private OracleVectorStoreQueryProvider( prefixForCollectionTables, buildSupportedKeyTypes(), buildSupportedDataTypes(stringTypeMapping, defaultVarcharSize), - buildSupportedVectorTypes(defaultVarcharSize)); + buildSupportedVectorTypes()); this.collectionsTable = collectionsTable; this.objectMapper = objectMapper; } private static HashMap, String> buildSupportedKeyTypes() { HashMap, String> supportedKeyTypes = new HashMap<>(); - supportedKeyTypes.put(String.class, "VARCHAR(255)"); + supportedKeyTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); + supportedKeyTypes.put(short.class, OracleDataTypesMapping.SHORT); + supportedKeyTypes.put(Short.class, OracleDataTypesMapping.SHORT); + supportedKeyTypes.put(int.class, OracleDataTypesMapping.INTEGER); + supportedKeyTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); + supportedKeyTypes.put(long.class, OracleDataTypesMapping.LONG); + supportedKeyTypes.put(Long.class, OracleDataTypesMapping.LONG); + supportedKeyTypes.put(UUID.class, OracleDataTypesMapping.UUID); + return supportedKeyTypes; } - private static Map, String> buildSupportedVectorTypes(int defaultVarCharLength) { + private static Map, String> buildSupportedVectorTypes() { HashMap, String> supportedVectorTypes = new HashMap<>(); supportedVectorTypes.put(String.class, "VECTOR(%s)"); supportedVectorTypes.put(List.class, "VECTOR(%s)"); @@ -102,22 +112,29 @@ private static Map, String> buildSupportedVectorTypes(int defaultVarCha private static Map, String> buildSupportedDataTypes(StringTypeMapping stringTypeMapping, int defaultVarCharLength) { HashMap, String> supportedDataTypes = new HashMap<>(); if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { - supportedDataTypes.put(String.class, "VARCHAR(" + defaultVarCharLength + ")"); + supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); } else { - supportedDataTypes.put(String.class, "CLOB"); + supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); } - supportedDataTypes.put(Integer.class, "INTEGER"); - supportedDataTypes.put(int.class, "INTEGER"); - supportedDataTypes.put(Long.class, "LONG"); - supportedDataTypes.put(long.class, "LONG"); - supportedDataTypes.put(Float.class, "REAL"); - supportedDataTypes.put(float.class, "REAL"); - supportedDataTypes.put(Double.class, "DOUBLE PRECISION"); - supportedDataTypes.put(double.class, "DOUBLE PRECISION"); - supportedDataTypes.put(Boolean.class, "BOOLEAN"); - supportedDataTypes.put(boolean.class, "BOOLEAN"); - supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); - supportedDataTypes.put(List.class, "JSON"); + supportedDataTypes.put(byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(Byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(Short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(int.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(Double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); + supportedDataTypes.put(Boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); + supportedDataTypes.put(UUID.class, OracleDataTypesMapping.UUID); + supportedDataTypes.put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); + supportedDataTypes.put(List.class, OracleDataTypesMapping.JSON); return supportedDataTypes; } @@ -178,11 +195,11 @@ public void createCollection(String collectionName, List vectorFields = recordDefinition.getVectorFields(); String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" - + "%s VARCHAR(255) PRIMARY KEY, " + + "%s PRIMARY KEY, " + "%s, " + "%s)", getCollectionTableName(collectionName), - getKeyColumnName(recordDefinition.getKeyField()), + getKeyColumnNameAndType(recordDefinition.getKeyField(), getSupportedDataTypes()), getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), getSupportedDataTypes()), getVectorColumnNamesAndTypes(new ArrayList<>(vectorFields), @@ -237,6 +254,11 @@ public void createCollection(String collectionName, } } + private String getKeyColumnNameAndType(VectorStoreRecordKeyField field, Map, String> types) { + return validateSQLidentifier(field.getEffectiveStorageName()) + " " + + types.get(field.getFieldType()); + } + private String createIndexForDataField(String collectionName, VectorStoreRecordDataField dataField) { if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; @@ -545,7 +567,7 @@ public static class Builder private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; private ObjectMapper objectMapper = new ObjectMapper(); private StringTypeMapping stringTypeMapping = StringTypeMapping.USE_VARCHAR; - private int defaultVarcharSize = 4000; + private int defaultVarcharSize = 2000; @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -594,7 +616,7 @@ public Builder withStringTypeMapping (StringTypeMapping stringTypeMapping) { /** * Sets the default size of the VARHCHAR2 fields. * @param defaultVarcharSize the default size of the VARHCHAR2 fields. By default, the size - * is 4000. + * is 2000. * @return then builder */ public Builder withDefaultVarcharSize (int defaultVarcharSize) { diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 1ba9fe7a..eaa8e7dc 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -147,29 +147,43 @@ public OracleVectorStoreRecordMapper build() { Object value; switch (supportedDataTypesMapping.get(fieldType)) { - case "CLOB": + case OracleDataTypesMapping.STRING_CLOB: value = resultSet.getString(field.getEffectiveStorageName()); break; - case "INTEGER": - value = resultSet.getInt(field.getEffectiveStorageName()); + case OracleDataTypesMapping.BYTE: + value = resultSet.getByte(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.SHORT: + value = resultSet.getShort(field.getEffectiveStorageName()); break; - case "LONG": + case OracleDataTypesMapping.INTEGER: value = resultSet.getInt(field.getEffectiveStorageName()); break; - case "REAL": + case OracleDataTypesMapping.LONG: + value = resultSet.getLong(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.FLOAT: value = resultSet.getFloat(field.getEffectiveStorageName()); break; - case "DOUBLE PRECISION": + case OracleDataTypesMapping.DOUBLE: value = resultSet.getDouble(field.getEffectiveStorageName()); break; - case "BOOLEAN": + case OracleDataTypesMapping.DECIMAL: + value = resultSet.getBigDecimal(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.BOOLEAN: value = resultSet.getBoolean(field.getEffectiveStorageName()); break; - case "TIMESTAMPTZ": + case OracleDataTypesMapping.OFFSET_DATE_TIME: value = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()) .offsetDateTimeValue(); break; - case "JSON": + case OracleDataTypesMapping.BYTE_ARRAY: + value = resultSet.getBytes(field.getEffectiveStorageName()); + break; + // fallthrough + case OracleDataTypesMapping.UUID: + case OracleDataTypesMapping.JSON: value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); break; default: From a9bc539b62e1e3ac76788308ac2b4db278327adf Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 17 Jun 2025 21:11:59 +0200 Subject: [PATCH 24/64] Refactoring # Conflicts: # data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java --- .../jdbc/JDBCVectorStoreRecordCollection.java | 6 +- .../jdbc/oracle/OracleDataTypesMapping.java | 4 + .../oracle/OracleVectorStoreFieldHelper.java | 307 ++++++++++++++++++ .../data/jdbc/oracle/Hotel.java | 22 +- ...OracleVectorStoreRecordCollectionTest.java | 25 +- .../VectorStoreRecordDefinition.java | 2 +- .../VectorStoreRecordVectorField.java | 6 +- 7 files changed, 350 insertions(+), 22 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java index 2d75af14..81395589 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -34,10 +34,10 @@ public class JDBCVectorStoreRecordCollection implements SQLVectorStoreRecordCollection { private final String collectionName; - private final VectorStoreRecordDefinition recordDefinition; - private final VectorStoreRecordMapper vectorStoreRecordMapper; + protected final VectorStoreRecordDefinition recordDefinition; + protected final VectorStoreRecordMapper vectorStoreRecordMapper; private final JDBCVectorStoreRecordCollectionOptions options; - private final SQLVectorStoreQueryProvider queryProvider; + protected final SQLVectorStoreQueryProvider queryProvider; /** * Creates a new instance of the {@link JDBCVectorStoreRecordCollection}. diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index f441a440..4b74527b 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -1,5 +1,8 @@ package com.microsoft.semantickernel.data.jdbc.oracle; +/** + * Defines oracle database type constants for supported field types. + */ public class OracleDataTypesMapping { public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; public static final String STRING_CLOB = "CLOB"; @@ -15,4 +18,5 @@ public class OracleDataTypesMapping { public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; public static final String UUID = "RAW(16)"; public static final String JSON = "JSON"; + public static final String VECTOR_FLOAT = "VECTOR(%s, FLOAT32)"; } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java new file mode 100644 index 00000000..3ae09ed4 --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -0,0 +1,307 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import oracle.jdbc.OracleTypes; +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Helper class for field operations. + */ +public class OracleVectorStoreFieldHelper { + private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); + + /** + * Maps supported key java classes to Oracle database types + */ + private static final HashMap, String> supportedKeyTypes = new HashMap() { + { + put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); + put(short.class, OracleDataTypesMapping.SHORT); + put(Short.class, OracleDataTypesMapping.SHORT); + put(int.class, OracleDataTypesMapping.INTEGER); + put(Integer.class, OracleDataTypesMapping.INTEGER); + put(long.class, OracleDataTypesMapping.LONG); + put(Long.class, OracleDataTypesMapping.LONG); + put(UUID .class, OracleDataTypesMapping.UUID); + } + }; + + /** + * Maps supported vector java classes to Oracle database types + */ + private static final Map, String> supportedVectorTypes = new HashMap() { + { + put(String.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(List.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); + put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); +/* + put(byte[].class,"VECTOR(%s, INT8)"); + put(Byte[].class,"VECTOR(%s, INT8)"); + put(double[].class,"VECTOR(%s, FLOAT64)"); + put(Double[].class,"VECTOR(%s, FLOAT64)"); + put(boolean[].class,"VECTOR(%s, BINARY)"); + put(Boolean[].class,"VECTOR(%s, BINARY)"); + */ + } + }; + + /** + * Maps supported data java classes to Oracle database types + */ + private static final HashMap, String> supportedDataTypes = new HashMap() { + { + put(byte.class, OracleDataTypesMapping.BYTE); + put(Byte.class, OracleDataTypesMapping.BYTE); + put(short.class, OracleDataTypesMapping.SHORT); + put(Short.class, OracleDataTypesMapping.SHORT); + put(int.class, OracleDataTypesMapping.INTEGER); + put(Integer.class, OracleDataTypesMapping.INTEGER); + put(long.class, OracleDataTypesMapping.LONG); + put(Long.class, OracleDataTypesMapping.LONG); + put(Float.class, OracleDataTypesMapping.FLOAT); + put(float.class, OracleDataTypesMapping.FLOAT); + put(Double.class, OracleDataTypesMapping.DOUBLE); + put(double.class, OracleDataTypesMapping.DOUBLE); + put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); + put(Boolean.class, OracleDataTypesMapping.BOOLEAN); + put(boolean.class, OracleDataTypesMapping.BOOLEAN); + put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); + put(UUID.class, OracleDataTypesMapping.UUID); + put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); + put(List.class, OracleDataTypesMapping.JSON); + } + + }; + + /** + * Maps vector type to OracleTypes. Only needed if types other than FLOAT_32 are supported. + */ + private static final Map, Integer> mapOracleTypeToVector = new HashMap() { + { + put(float[].class, OracleTypes.VECTOR_FLOAT32); + put(Float[].class, OracleTypes.VECTOR_FLOAT32); +/* + put(byte[].class, OracleTypes.VECTOR_INT8); + put(Byte[].class, OracleTypes.VECTOR_INT8); + put(Double[].class, OracleTypes.VECTOR_FLOAT64); + put(double[].class, OracleTypes.VECTOR_FLOAT64); + put(Boolean[].class, OracleTypes.VECTOR_BINARY); + put(boolean[].class, OracleTypes.VECTOR_BINARY); +*/ + } + }; + + /** + * Gets the mapping between the supported Java key types and the Oracle database type. + * + * @return the mapping between the supported Java key types and the Oracle database type. + */ + public static HashMap, String> getSupportedKeyTypes() { + return supportedKeyTypes; + } + + /** + * Gets the mapping between the supported Java data types and the Oracle database type. + * + * @return the mapping between the supported Java data types and the Oracle database type. + */ + public static Map, String> getSupportedDataTypes( + StringTypeMapping stringTypeMapping, int defaultVarCharLength) { + + if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { + supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); + } else { + supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); + } + return supportedDataTypes; + } + + /** + * Gets the mapping between the supported Java data types and the Oracle database type. + * + * @return the mapping between the supported Java data types and the Oracle database type. + */ + public static Map, String> getSupportedVectorTypes() { + return supportedVectorTypes; + } + + /** + * Generates the statement to create the index according to the vector field definition. + * + * @return the CREATE VECTOR INDEX statement to create the index according to the vector + * field definition. + */ + public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField field, String collectionTableName) { + switch (field.getIndexKind()) { + case IVFFLAT: + return "CREATE VECTOR INDEX IF NOT EXISTS " + + getIndexName(field.getEffectiveStorageName()) + + " ON " + + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + " ORGANIZATION NEIGHBOR PARTITIONS " + + " WITH DISTANCE COSINE " + + "PARAMETERS ( TYPE IVF )"; + case HNSW: + return "CREATE VECTOR INDEX IF NOT EXISTS " + getIndexName(field.getEffectiveStorageName()) + + " ON " + + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + "ORGANIZATION INMEMORY GRAPH " + + "WITH DISTANCE COSINE " + + "PARAMETERS (TYPE HNSW)"; + case UNDEFINED: + return null; + default: + LOGGER.warning("Unsupported index kind: " + field.getIndexKind()); + return null; + } + } + + /** + * Generates the statement to create the index according to the field definition. + * + * @return the CREATE INDEX statement to create the index according to the field definition. + */ + public static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { + if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { + String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; + return String.format(dataFieldIndex, + collectionTableName + "_" + dataField.getEffectiveStorageName(), + collectionTableName, + dataField.getEffectiveStorageName(), + getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); + } else { + String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; + return String.format(dataFieldIndex, + collectionTableName + "_" + dataField.getEffectiveStorageName(), + collectionTableName, + dataField.getEffectiveStorageName() + ); + } + } + + /** + * Gets the function that allows to return the function that converts the JSON value to the + * data type. + * @param jdbcType The JDBC type. + * @return the function that allows to return the function that converts the JSON value to the + * data type. + */ + private static String getFunctionForType(String jdbcType) { + switch (jdbcType) { + case OracleDataTypesMapping.BOOLEAN: + return "boolean()"; + case OracleDataTypesMapping.BYTE: + case OracleDataTypesMapping.SHORT: + case OracleDataTypesMapping.INTEGER: + case OracleDataTypesMapping.LONG: + case OracleDataTypesMapping.FLOAT: + case OracleDataTypesMapping.DOUBLE: + case OracleDataTypesMapping.DECIMAL: + return "numberOnly()"; + case OracleDataTypesMapping.OFFSET_DATE_TIME: + return "timestamp()"; + default: + return "string()"; + } + } + + /** + * Gets the type of the vector given the field definition. This method is not needed if only + * + * @param field the vector field definition. + * @return returns the type of vector for the given field type. + */ + public static String getTypeForVectorField(VectorStoreRecordVectorField field) { + String dimension = field.getDimensions() > 0 ? String.valueOf(field.getDimensions()) : "*"; + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); +/* Not needed since all types are FLOAT32 + if (field.getFieldSubType() != null) { + String vectorType; + switch (field.getFieldSubType().getName()) { + case "java.lang.Double": + vectorType = "FLOAT64"; + break; + case "java.lang.Byte": + vectorType = "INT8"; + break; + case "java.lang.Boolean": + vectorType = "BINARY"; + break; + default: + vectorType = "FLOAT32"; + } + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension, vectorType); + } else { + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); + } + */ + } + + /** + * Gets the JDBC oracle of the vector field definition. + * @param field the vector field definition. + * @return the JDBC oracle type. + */ + public static int getOracleTypeForField(VectorStoreRecordVectorField field) { + if (field.getFieldSubType() == null) { + return mapOracleTypeToVector.get(field.getFieldType()).intValue(); + } else { + switch (field.getFieldSubType().getName()) { + case "java.lang.Double": + return OracleTypes.VECTOR_FLOAT64; + case "java.lang.Byte": + return OracleTypes.VECTOR_INT8; + case "java.lang.Boolean": + return OracleTypes.VECTOR_BINARY; + default: + return OracleTypes.VECTOR_FLOAT32; + } + } + } + + /** + * Generates the index name given the field name. by suffixing "_VECTOR_INDEX" to the field name. + * @param effectiveStorageName the field name. + * @return the index name. + */ + private static String getIndexName(String effectiveStorageName) { + return effectiveStorageName + "_VECTOR_INDEX"; + } + + /** + * Returns vector columns names and types for CREATE TABLE statement + * @param fields list of vector record fields. + * @return comma separated list of columns and types for CREATE TABLE statement. + */ + public static String getVectorColumnNamesAndTypes(List fields) { + List columns = fields.stream() + .map(field -> field.getEffectiveStorageName() + " " + + OracleVectorStoreFieldHelper.getTypeForVectorField(field) + ).collect(Collectors.toList()); + + return String.join(", ", columns); + } + + /** + * Returns key column names and type for key column for CREATE TABLE statement + * @param field the key field. + * @return column name and type of the key field for CREATE TABLE statement. + */ + public static String getKeyColumnNameAndType(VectorStoreRecordKeyField field) { + return field.getEffectiveStorageName() + " " + supportedKeyTypes.get(field.getFieldType()); + } + +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index 197cb8c3..d1c848a6 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -40,15 +40,15 @@ public class Hotel { @JsonProperty("summaryEmbedding2") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) - private final List cosineDistance; + private final float[] cosineDistance; @JsonProperty("summaryEmbedding3") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY, indexKind = IndexKind.IVFFLAT) - private final List cosineSimilarity; + private final float[] cosineSimilarity; @JsonProperty("summaryEmbedding4") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT, indexKind = IndexKind.IVFFLAT) - private final List dotProduct; + private final Float[] dotProduct; @VectorStoreRecordData private double rating; @@ -66,9 +66,9 @@ protected Hotel( @JsonProperty("tags") List tags, @JsonProperty("summary") String description, @JsonProperty("summaryEmbedding1") List euclidean, - @JsonProperty("summaryEmbedding2") List cosineDistance, - @JsonProperty("summaryEmbedding3") List cosineSimilarity, - @JsonProperty("summaryEmbedding4") List dotProduct, + @JsonProperty("summaryEmbedding2") float[] cosineDistance, + @JsonProperty("summaryEmbedding3") float[] cosineSimilarity, + @JsonProperty("summaryEmbedding4") Float[] dotProduct, @JsonProperty("rating") double rating) { this.id = id; this.name = name; @@ -77,9 +77,9 @@ protected Hotel( this.tags = tags; this.description = description; this.euclidean = euclidean; - this.cosineDistance = euclidean; - this.cosineSimilarity = euclidean; - this.dotProduct = euclidean; + this.cosineDistance = cosineDistance; + this.cosineSimilarity = cosineSimilarity; + this.dotProduct = dotProduct; this.rating = rating; } @@ -107,11 +107,11 @@ public List getEuclidean() { return euclidean; } - public List getCosineDistance() { + public float[] getCosineDistance() { return cosineDistance; } - public List getDotProduct() { + public Float[] getDotProduct() { return dotProduct; } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 72ed3524..22d0259f 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -128,19 +128,34 @@ public void clearCollection() { private static List getHotels() { return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", - Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), + new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, + new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, + new Float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, 4.0), new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", - Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, + Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), + new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, + new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, + new Float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, 4.0), new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", - Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, + Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), + new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, + new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, + new Float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, 5.0), new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", - Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, + Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), + new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, + new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, + new Float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, 4.0), new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", - Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), + new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, + new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, + new Float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, 4.0)); } diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 8f8d6212..e769bb6f 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -210,7 +210,7 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) vectorFields.add(VectorStoreRecordVectorField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType()) + .withFieldType(field.getType(), List.class.equals(field.getType()) ? (Class)((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0] : null) .withDimensions(vectorAttribute.dimensions()) .withIndexKind(vectorAttribute.indexKind()) .withDistanceFunction(vectorAttribute.distanceFunction()) diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java index 00b7627a..b708d2fb 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java @@ -34,10 +34,11 @@ public VectorStoreRecordVectorField( @Nonnull String name, @Nullable String storageName, @Nonnull Class fieldType, + Class fieldSubType, int dimensions, @Nullable IndexKind indexKind, @Nullable DistanceFunction distanceFunction) { - super(name, storageName, fieldType); + super(name, storageName, fieldType, fieldSubType); this.dimensions = dimensions; this.indexKind = indexKind == null ? IndexKind.UNDEFINED : indexKind; this.distanceFunction = distanceFunction == null ? DistanceFunction.UNDEFINED @@ -130,7 +131,8 @@ public VectorStoreRecordVectorField build() { throw new IllegalArgumentException("dimensions must be greater than 0"); } - return new VectorStoreRecordVectorField(name, storageName, fieldType, dimensions, + return new VectorStoreRecordVectorField(name, storageName, fieldType, fieldSubType, + dimensions, indexKind, distanceFunction); } From 17fce47b46131503a7c5c4d8874f46d8576013cb Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 7 Jun 2025 00:52:21 -0700 Subject: [PATCH 25/64] Add test for IndexKind.UNDEFINED and IndexKind.HNSW --- ...OracleVectorStoreRecordCollectionTest.java | 161 +++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 22d0259f..06e8c8d6 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -9,17 +9,28 @@ import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import oracle.jdbc.OracleConnection; import oracle.jdbc.datasource.impl.OracleDataSource; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.time.Duration; import java.util.Arrays; import java.util.List; @@ -42,7 +53,6 @@ public class OracleVectorStoreRecordCollectionTest { private static final OracleDataSource DATA_SOURCE; private static final OracleDataSource SYSDBA_DATA_SOURCE; - static { try { @@ -339,6 +349,155 @@ public void searchWithTagFilter() { assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); } + @Nested + class HNSWIndexTests { + @Test + void testHNSWIndexIsCreatedSuccessfully() throws Exception { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .isFilterable(false) + .build(); + + VectorStoreRecordVectorField hnswVector= VectorStoreRecordVectorField.builder() + .withName("hnsw") + .withStorageName("hnsw") + .withFieldType(List.class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_SIMILARITY) + .withIndexKind(IndexKind.HNSW) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, hnswVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "skhotels_hnsw"; + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Object.class) + .withRecordDefinition(definition) + .build()); + + // create collection + collection.createCollectionAsync().block(); + + String expectedIndexName = hnswVector.getEffectiveStorageName().toUpperCase() + "_VECTOR_INDEX"; + + // check if index exist + try (Connection conn = DATA_SOURCE.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "SELECT COUNT(*) FROM USER_INDEXES WHERE INDEX_NAME=?")) { + stmt.setString(1, expectedIndexName); + ResultSet rs = stmt.executeQuery(); + rs.next(); + int count = rs.getInt(1); + + assertEquals(1, count, "hnsw vector index should have been created"); + } finally { + // clean up + try (Connection conn = DATA_SOURCE.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE " + "SKCOLLECTION_" + collectionName); + } + } + } + } + + @Nested + class UndefinedIndexTests { + @Test + void testNoIndexIsCreatedForUndefined() throws Exception { + // create key field + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + // create vector field, set IndexKind to UNDEFINED + VectorStoreRecordVectorField undefinedVector= VectorStoreRecordVectorField.builder() + .withName("undef") + .withStorageName("undef") + .withFieldType(List.class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_SIMILARITY) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .isFilterable(false) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, undefinedVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "skhotels_undefined"; + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Object.class) + .withRecordDefinition(definition) + .build()); + + // create collection + collection.createCollectionAsync().block(); + + // check if index exist + String expectedIndexName = undefinedVector.getEffectiveStorageName().toUpperCase() + "_VETCOR_INDEX"; + try (Connection conn = DATA_SOURCE.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "SELECT COUNT(*) FROM USER_INDEXES WHERE INDEX_NAME = ?")) { + stmt.setString(1, expectedIndexName); + ResultSet rs = stmt.executeQuery(); + rs.next(); + int count = rs.getInt(1); + + assertEquals(0,count,"Vector index should not be created for IndexKind.UNDEFINED"); + } finally { + // clean up + try (Connection conn = DATA_SOURCE.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE " + "SKCOLLECTION_" + collectionName); + } + } + } + } + private static Stream distanceFunctionAndDistance() { return Stream.of( Arguments.of (DistanceFunction.COSINE_DISTANCE, 0.8548d), From a9e82132cd15f50eaa09b30a19fac7449a1946ca Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 14 Jun 2025 00:50:03 -0700 Subject: [PATCH 26/64] add test for key type --- ...OracleVectorStoreRecordCollectionTest.java | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 06e8c8d6..8ad16702 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -1,5 +1,7 @@ package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; @@ -16,6 +18,7 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; + import oracle.jdbc.OracleConnection; import oracle.jdbc.datasource.impl.OracleDataSource; import org.junit.jupiter.api.BeforeAll; @@ -33,7 +36,9 @@ import java.sql.Statement; import java.time.Duration; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -349,6 +354,69 @@ public void searchWithTagFilter() { assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); } + @ParameterizedTest + @MethodSource("supportedKeyTypes") + void testKeyTypes(String suffix, Class keyType, Object keyValue) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(keyType) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .build(); + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vec") + .withStorageName("vec") + .withFieldType(List.class) + .withDimensions(2) + .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_keytype_" + suffix; + + VectorStoreRecordCollection collectionRaw = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(DummyRecordForKeyTypes.class) + .withRecordDefinition(definition) + .build()); + + VectorStoreRecordCollection collection = + (VectorStoreRecordCollection) collectionRaw; + + collection.createCollectionAsync().block(); + + DummyRecordForKeyTypes record = new DummyRecordForKeyTypes(keyValue, "dummyValue", Arrays.asList(1.0f, 2.0f)); + collection.upsertAsync(record, null).block(); + + DummyRecordForKeyTypes result = collection.getAsync(keyValue, null).block(); + assertNotNull(result); + assertEquals("dummyValue", result.getDummy()); + + collection.deleteCollectionAsync().block(); + } + @Nested class HNSWIndexTests { @Test @@ -517,4 +585,48 @@ private static Stream parametersExactSearch() { Arguments.of (DistanceFunction.UNDEFINED, Arrays.asList(0.1000d, 18.9081d, 19.9669d)) ); } + + // commented out temporarily because only String type key is supported in + // JDBCVectorStoreRecordCollection#getKeyFromRecord: + // ... + // return (String) keyField.get(data); + // ... + // thus upsertAync/getAsync won't work + private static Stream supportedKeyTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123")/*, + Arguments.of("integer", Integer.class, 321), + Arguments.of("long", Long.class, 5L), + Arguments.of("short", Short.class, (short) 3), + Arguments.of("uuid", UUID.class, UUID.randomUUID())*/ + ); + } + + private static class DummyRecordForKeyTypes { + private final Object id; + private final String dummy; + private final List vec; + @JsonCreator + public DummyRecordForKeyTypes( + @JsonProperty("id")Object id, + @JsonProperty("dummy") String dummy, + @JsonProperty("vec") List vec) { + this.id = id; + this.dummy = dummy; + this.vec = vec; + } + + public Object getId() { + return id; + } + + public String getDummy() { + return dummy; + } + + @Override + public String toString() { + return String.valueOf(id); + } + } } From b3e1f4bf646b56326d12213c1a62ea996894206c Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Tue, 17 Jun 2025 23:17:44 -0700 Subject: [PATCH 27/64] Add test for supported data type(timestamp and uuid not included) --- ...OracleVectorStoreRecordCollectionTest.java | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 8ad16702..20d49292 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -29,20 +29,25 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.time.Duration; +import java.time.OffsetDateTime; import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -417,6 +422,87 @@ void testKeyTypes(String suffix, Class keyType, Object keyValue) { collection.deleteCollectionAsync().block(); } + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFieldValue, Class fieldSubType) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dataField; + if (fieldSubType != null) { + dataField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(dataFieldType, fieldSubType) + .isFilterable(true) + .build(); + } else { + dataField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(dataFieldType) + .isFilterable(true) + .build(); + } + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vec") + .withStorageName("vec") + .withFieldType(List.class) + .withDimensions(2) + .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dataField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_datatype_" + dataFieldName; + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions. builder() + .withRecordClass(DummyRecordForDataTypes.class) + .withRecordDefinition(definition).build()); + + collection.createCollectionAsync().block(); + + String key = "testid"; + + DummyRecordForDataTypes record = + new DummyRecordForDataTypes(key, dataFieldValue, Arrays.asList(1.0f, 2.0f)); + + collection.upsertAsync(record, null).block(); + + DummyRecordForDataTypes result = collection.getAsync(key, null).block(); + assertNotNull(result); + + if (dataFieldValue instanceof Number && result.getDummy() instanceof Number) { + assertEquals(((Number) dataFieldValue).doubleValue(), ((Number) result.getDummy()).doubleValue()); + } else if (dataFieldValue instanceof byte[]) { + assertArrayEquals((byte[]) dataFieldValue, (byte[]) result.getDummy()); + } else { + assertEquals(dataFieldValue, result.getDummy()); + } + + collection.deleteCollectionAsync().block(); + } + @Nested class HNSWIndexTests { @Test @@ -602,6 +688,25 @@ private static Stream supportedKeyTypes() { ); } + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123", null), + Arguments.of("boolean_true", Boolean.class, true, null), + Arguments.of("boolean_false", Boolean.class, false, null), + Arguments.of("byte", Byte.class, (byte) 127, null), + Arguments.of("short", Short.class, (short) 3, null), + Arguments.of("integer", Integer.class, 321, null), + Arguments.of("long", Long.class, 5L, null), + Arguments.of("float", Float.class, 3.14f, null), + Arguments.of("double", double.class, 3.14159265358d, null), + Arguments.of("decimal", BigDecimal.class, new BigDecimal("12345.67"), null), + //Arguments.of("timestamp", OffsetDateTime.class, OffsetDateTime.now(), null) + //Arguments.of("uuid", UUID.class, UUID.randomUUID(), null) + Arguments.of("byte_array", byte[].class, "abc".getBytes(StandardCharsets.UTF_8), null), + Arguments.of("json", List.class, Arrays.asList("a", "s", "d"), String.class) + ); + } + private static class DummyRecordForKeyTypes { private final Object id; private final String dummy; @@ -629,4 +734,32 @@ public String toString() { return String.valueOf(id); } } + + private static class DummyRecordForDataTypes { + private final String id; + private final Object dummy; + private final List vec; + @JsonCreator + public DummyRecordForDataTypes( + @JsonProperty("id") String id, + @JsonProperty("dummy") Object dummy, + @JsonProperty("vec") List vec) { + this.id = id; + this.dummy = dummy; + this.vec = vec; + } + + public String getId() { + return id; + } + + public Object getDummy() { + return dummy; + } + + @Override + public String toString() { + return String.valueOf(id); + } + } } From ec313762fe796b5a5da0e9c50db5cf0eef0803ce Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Fri, 20 Jun 2025 18:46:42 +0200 Subject: [PATCH 28/64] Test and bug fixes --- .../OracleVectorStoreDataTypeSearchTest.java | 297 ++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java new file mode 100644 index 00000000..4ce739a8 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -0,0 +1,297 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OracleVectorStoreDataTypeSearchTest extends OracleCommonVectorStoreRecordCollectionTest { + private static final double MIN_NUMBER = 1.0E-130; + private static final BigDecimal BIG_NUMBER = BigDecimal.valueOf(9999999999999999.99); + + + + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypesSearch (ClassWithAllBoxedTypes record) { + VectorStoreRecordCollection collection = setupBoxed(); + + collection.upsertAsync(record, null).block(); + + // boolean + VectorSearchResults results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("booleanValue", record.getBooleanValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getBooleanValue(), results.getResults().get(0).getRecord().getBooleanValue()); + + // byte + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("byteValue", record.getByteValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getByteValue(), results.getResults().get(0).getRecord().getByteValue()); + + // short + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("shortValue", record.getShortValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getShortValue(), results.getResults().get(0).getRecord().getShortValue()); + + // integer + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("integerValue", record.getIntegerValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getIntegerValue(), results.getResults().get(0).getRecord().getIntegerValue()); + + // long + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("longValue", record.getLongValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getLongValue(), results.getResults().get(0).getRecord().getLongValue()); + + // float + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("floatValue", record.getFloatValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getFloatValue(), results.getResults().get(0).getRecord().getFloatValue()); + + // double + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("doubleValue", record.getDoubleValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); + + // decimal + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("decimalValue", record.getDecimalValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + if (record.getDecimalValue() != null) { + assertEquals(0, record.getDecimalValue() + .compareTo(results.getResults().get(0).getRecord().getDecimalValue())); + } else { + assertEquals(record.getDecimalValue(), + results.getResults().get(0).getRecord().getDecimalValue()); + } + + // offset date time + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("offsetDateTimeValue", record.getOffsetDateTimeValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + if (record.getOffsetDateTimeValue() != null) { + assertTrue(record.getOffsetDateTimeValue() + .isEqual(results.getResults().get(0).getRecord().getOffsetDateTimeValue())); + } else { + assertEquals(record.getOffsetDateTimeValue(), + results.getResults().get(0).getRecord().getOffsetDateTimeValue()); + } + + // UUID + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("uuidValue", record.getUuidValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getUuidValue(), results.getResults().get(0).getRecord().getUuidValue()); + + // byte array + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("byteArrayValue", record.getByteArrayValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertArrayEquals(record.getByteArrayValue(), results.getResults().get(0).getRecord().getByteArrayValue()); + + collection.deleteCollectionAsync().block(); + + } + + + public VectorStoreRecordCollection setupBoxed() { + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = + vectorStore.getCollection("BoxedTypes", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(ClassWithAllBoxedTypes.class) + .build()).createCollectionAsync().block(); + + collection.createCollectionAsync().block(); + + return collection; + } + + + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllBoxedTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, + Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + OffsetDateTime.now(), UUID.randomUUID(), new byte[] {Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", null, null, null, null, null, + null, null, null, + null, null, null, + null, + null + ) + ) + ); + } + + private static Stream supportedDataPrimitiveTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, + Long.MIN_VALUE, + Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + OffsetDateTime.now(), UUID.randomUUID(), + new byte[]{Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, + Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), + BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, (byte) 0, (short) 0, 0, 0l, + 0f, 0d, null, + null, null, null, + null, + null + ) + ) + ); + } +} From cb4f72692c7e6be0c139d53524c9d2198edb2287 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Sat, 21 Jun 2025 00:20:02 +0200 Subject: [PATCH 29/64] BigDecimal mapping and value range --- .../data/jdbc/oracle/OracleDataTypesMapping.java | 2 +- .../OracleVectorStoreDataTypeSearchTest.java | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index 4b74527b..0258cb5f 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -14,7 +14,7 @@ public class OracleDataTypesMapping { public static final String LONG = "NUMBER(19)"; public static final String FLOAT = "BINARY_FLOAT"; public static final String DOUBLE = "BINARY_DOUBLE"; - public static final String DECIMAL = "NUMBER(18,2)"; + public static final String DECIMAL = "NUMBER"; public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; public static final String UUID = "RAW(16)"; public static final String JSON = "JSON"; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java index 4ce739a8..d99234ab 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -4,11 +4,9 @@ import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -24,7 +22,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class OracleVectorStoreDataTypeSearchTest extends OracleCommonVectorStoreRecordCollectionTest { - private static final double MIN_NUMBER = 1.0E-130; + private static final double MIN_DOUBLE = 1.0E-130; + private static final double MIN_DECIMAL = -1.0E125; private static final BigDecimal BIG_NUMBER = BigDecimal.valueOf(9999999999999999.99); @@ -120,6 +119,11 @@ void testDataTypesSearch (ClassWithAllBoxedTypes record) { assertEquals(1, results.getTotalCount()); assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); + System.out.println(record.getDecimalValue()); + System.out.println(record.getDecimalValue().doubleValue()); + System.out.println(results.getResults().get(0).getRecord().getDecimalValue()); + System.out.println(results.getResults().get(0).getRecord().getDecimalValue().doubleValue()); + // decimal results = collection.searchAsync( null, @@ -223,7 +227,7 @@ private static Stream supportedDataTypes() { Arguments.of( new ClassWithAllBoxedTypes( "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, - Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + Float.MIN_VALUE, MIN_DOUBLE, BigDecimal.valueOf(MIN_DECIMAL), OffsetDateTime.now(), UUID.randomUUID(), new byte[] {Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } @@ -265,7 +269,7 @@ private static Stream supportedDataPrimitiveTypes() { new ClassWithAllPrimitiveTypes( "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, - Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + Float.MIN_VALUE, MIN_DOUBLE, BigDecimal.valueOf(MIN_DECIMAL), OffsetDateTime.now(), UUID.randomUUID(), new byte[]{Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), From 6bd3a8e646a9c8383487ea3f4548b1e37d3e1687 Mon Sep 17 00:00:00 2001 From: psilberk Date: Fri, 25 Apr 2025 16:20:54 -0700 Subject: [PATCH 30/64] Upsert and Main sample --- data/semantickernel-data-jdbc/pom.xml | 43 +++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 7cd77f3f..1f2e1a28 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-SNAPSHOT ../../pom.xml @@ -14,12 +14,13 @@ - org.slf4j - slf4j-api + com.microsoft.semantic-kernel + semantickernel-api + - com.microsoft.semantic-kernel - semantickernel-api-data + org.slf4j + slf4j-api com.fasterxml.jackson.core @@ -31,9 +32,41 @@ jackson-core compile + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + com.github.spotbugs spotbugs-annotations + + org.apache.commons + commons-text + + + org.postgresql + postgresql + 42.7.4 + + + org.xerial + sqlite-jdbc + 3.47.0.0 + + + com.oracle.database.jdbc + ojdbc11 + 23.7.0.25.01 + \ No newline at end of file From 2e3133390f8d808e6e7bf1abf9261fc20ce36bc9 Mon Sep 17 00:00:00 2001 From: psilberk Date: Mon, 23 Jun 2025 14:57:40 -0700 Subject: [PATCH 31/64] Books demo with vector similarity search --- .../data/vectorstores/oracle/Book.java | 49 ++++++++++++++----- .../data/vectorstores/oracle/Main.java | 26 +++++++--- 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java index aeaff6e8..87104cf8 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java @@ -4,12 +4,12 @@ import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; -import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; -import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import java.util.List; public class Book { + public Book() {} + public Book(String isbn, String title, String author, int pages, List tags, String summary, List summaryEmbedding) { this.isbn = isbn; @@ -21,27 +21,26 @@ public Book(String isbn, String title, String author, int pages, this.summaryEmbedding = summaryEmbedding; } - @VectorStoreRecordKey - private final String isbn; + private String isbn; @VectorStoreRecordData(isFilterable = true) - private final String title; + private String title; @VectorStoreRecordData(isFilterable = true) - private final String author; + private String author; @VectorStoreRecordData - private final int pages; + private int pages; @VectorStoreRecordData(isFilterable = true) - private final List tags; + private List tags; @VectorStoreRecordData( isFilterable = true, isFullTextSearchable = true ) - private final String summary; + private String summary; - @VectorStoreRecordVector(dimensions = 4, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) - private final List summaryEmbedding; + @VectorStoreRecordVector(dimensions = 2) + private List summaryEmbedding; public String getIsbn() { return isbn; @@ -70,4 +69,32 @@ public String getSummary() { public List getSummaryEmbedding() { return summaryEmbedding; } + + public void setIsbn(String isbn) { + this.isbn = isbn; + } + + public void setTitle(String title) { + this.title = title; + } + + public void setAuthor(String author) { + this.author = author; + } + + public void setPages(int pages) { + this.pages = pages; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public void setSummaryEmbedding(List summaryEmbedding) { + this.summaryEmbedding = summaryEmbedding; + } + + public void setSummary(String summary) { + this.summary = summary; + } } \ No newline at end of file diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java index d71a0f7e..babc381f 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -5,11 +5,14 @@ import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import java.sql.SQLException; import java.util.Arrays; import java.util.List; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import oracle.jdbc.datasource.impl.OracleDataSource; +import reactor.core.publisher.Mono; public class Main { public static void main(String[] args) throws SQLException { @@ -45,23 +48,30 @@ public static void main(String[] args) throws SQLException { collection.upsertBatchAsync(books, null).block(); // Retrieve the upserted record. - //var retrievedBook = collection.getAsync("1", null).block(); + Book retrievedBook = collection.getAsync("2", null).block(); + + System.out.println(retrievedBook.getAuthor()); // Generate a vector for your search text, using your chosen embedding generation implementation. // Just showing a placeholder method here for brevity. - // var searchVector = generateEmbeddingsAsync( - // "I'm looking for a Book where customer happiness is the priority.").block(); + List searchVector = generateEmbeddingsAsync( + "I'm looking for a horror book.").block(); // Do the search. - // var searchResult = collection.searchAsync(searchVector, VectorSearchOptions.builder() - // .withTop(1).build()).block(); + VectorSearchResults searchResult = collection.searchAsync( + searchVector, VectorSearchOptions.builder().withTop(1).build()).block(); - // Book record = searchResult.getResults().get(0).getRecord(); - // System.out.printf("Found Book description: %s\n", record.getDescription()); + retrievedBook = searchResult.getResults().get(0).getRecord(); + System.out.println("Found Book: " + retrievedBook.getIsbn()); } static List books = Arrays.asList( - new Book("1", "one", "sking", 0, null, "sum", null)); + new Book("1", "one", "sking", 0, null, "horror", List.of(1f, 1f)), + new Book("2", "two", "squeen", 0, null, "non-fiction", List.of(-11f, -11f))); + + private static Mono> generateEmbeddingsAsync(String text) { + return Mono.just(List.of(-0.9f, -0.9f)); + } } \ No newline at end of file From 32f3ddce32d65d9520cf139c1d581d87c591153b Mon Sep 17 00:00:00 2001 From: psilberk Date: Mon, 23 Jun 2025 15:44:34 -0700 Subject: [PATCH 32/64] Fix pom for demo --- .../semantickernel-syntax-examples/pom.xml | 12 ++++++++++++ samples/semantickernel-learn-resources/pom.xml | 6 ++++++ .../data/vectorstores/oracle/Main.java | 4 ++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 35efaffc..d5471730 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -147,6 +147,18 @@ 1.4.4-SNAPSHOT compile + + com.microsoft.semantic-kernel + semantickernel-data-postgres + 1.4.4-RC2-SNAPSHOT + compile + + + com.microsoft.semantic-kernel + semantickernel-data-oracle + 1.4.4-RC2-SNAPSHOT + compile + diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index b8a1ea0b..8d3cef89 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -94,6 +94,12 @@ 9.0.0 compile + + com.microsoft.semantic-kernel + semantickernel-data-postgres + 1.4.4-RC2-SNAPSHOT + compile + diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java index babc381f..570dfc60 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -68,10 +68,10 @@ public static void main(String[] args) throws SQLException { static List books = Arrays.asList( new Book("1", "one", "sking", 0, null, "horror", List.of(1f, 1f)), - new Book("2", "two", "squeen", 0, null, "non-fiction", List.of(-11f, -11f))); + new Book("2", "two", "squeen", 0, null, "non-fiction", List.of(-1f, -1f))); private static Mono> generateEmbeddingsAsync(String text) { - return Mono.just(List.of(-0.9f, -0.9f)); + return Mono.just(List.of(-0.1f, -0.1f)); } } \ No newline at end of file From b7d5ab3102ccadf9fe52672e1174627fb83aee70 Mon Sep 17 00:00:00 2001 From: psilberk Date: Tue, 24 Jun 2025 08:12:45 -0700 Subject: [PATCH 33/64] Fixing samples poms --- .../semantickernel-syntax-examples/pom.xml | 6 ++++++ samples/semantickernel-learn-resources/pom.xml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index ffc2adae..53b09d38 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -136,6 +136,12 @@ com.github.victools jsonschema-module-jackson + + com.microsoft.semantic-kernel + semantickernel-data-postgres + 1.4.4-RC2-SNAPSHOT + compile + diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 68062bf4..9d0cdb1b 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -85,6 +85,12 @@ 9.0.0 compile + + com.microsoft.semantic-kernel + semantickernel-data-postgres + 1.4.4-RC2-SNAPSHOT + compile + From 67aec2b3a8c317a11ff88dd0909e14e69df01990 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Wed, 25 Jun 2025 20:06:27 +0200 Subject: [PATCH 34/64] JsonTypeInfo annotation --- .../OracleVectorStoreQueryProvider.java | 18 +- .../oracle/OracleVectorStoreRecordMapper.java | 28 +++ .../jdbc/oracle/ClassWithAnnotatedTypes.java | 65 ++++++ .../OracleVectorStoreAnnotatedTypeTest.java | 200 ++++++++++++++++++ 4 files changed, 309 insertions(+), 2 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index f7a32fd5..fa606e64 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -51,6 +51,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -67,6 +68,8 @@ public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider // This could be common to all query providers private final ObjectMapper objectMapper; + private final Map, String> annotatedTypeMapping; + private static final Object dbCreationLock = new Object(); private static final Logger logger = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); @@ -97,7 +100,8 @@ private OracleVectorStoreQueryProvider( @Nonnull String prefixForCollectionTables, int defaultVarcharSize, @Nonnull StringTypeMapping stringTypeMapping, - ObjectMapper objectMapper) { + ObjectMapper objectMapper, + Map, String> annotatedTypeMapping) { super( dataSource, collectionsTable, @@ -108,6 +112,7 @@ private OracleVectorStoreQueryProvider( this.collectionsTable = collectionsTable; this.objectMapper = objectMapper; this.objectMapper.registerModule(new JavaTimeModule()); + this.annotatedTypeMapping = annotatedTypeMapping; } @Override @@ -569,6 +574,7 @@ public VectorStoreRecordMapper getVectorStoreRecordM .withRecordClass(recordClass) .withVectorStoreRecordDefinition(vectorStoreRecordDefinition) .withSupportedDataTypesMapping(getSupportedDataTypes()) + .withAnnotatedTypeMapping(annotatedTypeMapping) .build(); } @@ -586,6 +592,8 @@ public static class Builder private StringTypeMapping stringTypeMapping = StringTypeMapping.USE_VARCHAR; private int defaultVarcharSize = 2000; + private Map, String> annotatedTypeMapping = null; + @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withDataSource(DataSource dataSource) { @@ -619,6 +627,11 @@ public Builder withObjectMapper( return this; } + public Builder withAnnotatedTypeMapping(Map, String> annotatedTypeMapping) { + this.annotatedTypeMapping = annotatedTypeMapping; + return this; + } + /** * Sets the desired String type mapping. * @param stringTypeMapping the desired String type mapping. The default value is @@ -644,7 +657,8 @@ public Builder withDefaultVarcharSize (int defaultVarcharSize) { @Override public OracleVectorStoreQueryProvider build() { return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, - prefixForCollectionTables, defaultVarcharSize, stringTypeMapping, objectMapper); + prefixForCollectionTables, defaultVarcharSize, stringTypeMapping, objectMapper, + annotatedTypeMapping); } } } \ No newline at end of file diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index a7b42745..e6e7bf0a 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -1,9 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -69,6 +71,7 @@ public static class Builder private VectorStoreRecordDefinition vectorStoreRecordDefinition; private Map, String> supportedDataTypesMapping; private ObjectMapper objectMapper = new ObjectMapper(); + private Map, String> annotatedTypeMapping; /** * Sets the record class. @@ -118,6 +121,11 @@ public Builder withSupportedDataTypesMapping( return this; } + public Builder withAnnotatedTypeMapping(Map, String> annotatedTypeMapping) { + this.annotatedTypeMapping = annotatedTypeMapping; + return this; + } + /** * Builds the {@link OracleVectorStoreRecordMapper}. * @@ -142,6 +150,14 @@ public OracleVectorStoreRecordMapper build() { for (VectorStoreRecordField field : vectorStoreRecordDefinition.getNonVectorFields()) { Class fieldType = field.getFieldType(); + boolean isAnnotated = false; + try { + isAnnotated = recordClass.getDeclaredField(field.getName()) + .isAnnotationPresent(JsonTypeInfo.class); + } catch (NoSuchFieldException e) { + // ignore exception, assume the field is not annotated + } + Object value; switch (supportedDataTypesMapping.get(fieldType)) { case OracleDataTypesMapping.STRING_CLOB: @@ -204,6 +220,18 @@ public OracleVectorStoreRecordMapper build() { JsonNode genericNode = objectMapper.valueToTree(value); objectNode.set(field.getEffectiveStorageName(), genericNode); + if (isAnnotated) { + if (annotatedTypeMapping != null && annotatedTypeMapping.containsKey(field.getFieldType())) { + objectNode.set(field.getEffectiveStorageName() + "_type", + TextNode.valueOf( + annotatedTypeMapping.get(field.getFieldType()))); + } else { + objectNode.set(field.getEffectiveStorageName() + "_type", + TextNode.valueOf( + field.getFieldType().getName())); + + } + } } if (options != null && options.isIncludeVectors()) { for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java new file mode 100644 index 00000000..318911d8 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java @@ -0,0 +1,65 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; +import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; + +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.UUID; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ClassWithAnnotatedTypes { + + private final String id; + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = As.EXTERNAL_PROPERTY , property = "value_type") + @JsonSubTypes({ + @JsonSubTypes.Type(value = String.class, name="java.lang.String"), + @JsonSubTypes.Type(value = Boolean.class, name="java.lang.Boolean"), + @JsonSubTypes.Type(value = Byte.class, name="java.lang.Byte"), + @JsonSubTypes.Type(value = Short.class, name="java.lang.Short"), + @JsonSubTypes.Type(value = Integer.class, name="java.lang.Integer"), + @JsonSubTypes.Type(value = Long.class, name="java.lang.Long"), + @JsonSubTypes.Type(value = Float.class, name="java.lang.Float"), + @JsonSubTypes.Type(value = Double.class, name="java.lang.Double"), + @JsonSubTypes.Type(value = BigDecimal.class, name="java.math.BigDecimal"), + @JsonSubTypes.Type(value = OffsetDateTime.class, name="java.time.OffsetDateTime"), + @JsonSubTypes.Type(value = UUID.class, name="java.util.UUID"), + @JsonSubTypes.Type(value = byte[].class, name="byte_array"), + @JsonSubTypes.Type(value = List.class, name="listOfStrings") + }) + private Object value; + + private final Float[] vectorValue; + + + public ClassWithAnnotatedTypes() { + this(null, null, null); + }; + public ClassWithAnnotatedTypes(String id, Object value, Float[] vectorValue) { + this.id = id; + this.value = value; + this.vectorValue = vectorValue; + } + + public String getId() { + return id; + } + + public Object getValue() { + return value; + } + + public Float[] getVectorValue() { + return vectorValue; + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java new file mode 100644 index 00000000..9650652e --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java @@ -0,0 +1,200 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +public class OracleVectorStoreAnnotatedTypeTest extends OracleCommonVectorStoreRecordCollectionTest { + + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFieldValue, Class fieldSubType) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dataField; + if (fieldSubType != null) { + dataField = VectorStoreRecordDataField.builder() + .withName("value") + .withStorageName("value") + .withFieldType(dataFieldType, fieldSubType) + .isFilterable(true) + .build(); + } else { + dataField = VectorStoreRecordDataField.builder() + .withName("value") + .withStorageName("value") + .withFieldType(dataFieldType) + .isFilterable(true) + .build(); + } + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vectorValue") + .withStorageName("vectorValue") + .withFieldType(Float[].class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_DISTANCE) + .withIndexKind(IndexKind.IVFFLAT) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dataField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .withAnnotatedTypeMapping(new HashMap() {{ + put(List.class, "listOfStrings"); + }}) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_datatype_" + dataFieldName; + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions. builder() + .withRecordClass(ClassWithAnnotatedTypes.class) + .withRecordDefinition(definition).build()); + + collection.createCollectionAsync().block(); + + String key = "testid"; + + ClassWithAnnotatedTypes record = + new ClassWithAnnotatedTypes(key, dataFieldValue, new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }); + + collection.upsertAsync(record, null).block(); + + ClassWithAnnotatedTypes result = collection.getAsync(key, null).block(); + assertNotNull(result); + if (record.getValue().getClass().equals(OffsetDateTime.class)) { + assertTrue(((OffsetDateTime)dataFieldValue).isEqual((OffsetDateTime)record.getValue())); + } else { + assertEquals(dataFieldValue, result.getValue()); + } + + collection.deleteCollectionAsync().block(); + } + + @Test + void testByteArrayType() { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dataField; + dataField = VectorStoreRecordDataField.builder() + .withName("value") + .withStorageName("value") + .withFieldType(byte[].class) + .isFilterable(true) + .build(); + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vectorValue") + .withStorageName("vectorValue") + .withFieldType(Float[].class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_DISTANCE) + .withIndexKind(IndexKind.IVFFLAT) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dataField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .withAnnotatedTypeMapping(new HashMap() {{ + put(byte[].class, "byte_array"); + }}) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_datatype_bytearray"; + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions. builder() + .withRecordClass(ClassWithAnnotatedTypes.class) + .withRecordDefinition(definition).build()); + + collection.createCollectionAsync().block(); + + String key = "testid"; + + byte[] dataFieldValue = new byte[] {1, 2, 3}; + ClassWithAnnotatedTypes record = + new ClassWithAnnotatedTypes(key, dataFieldValue, new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }); + + collection.upsertAsync(record, null).block(); + + ClassWithAnnotatedTypes result = collection.getAsync(key, null).block(); + assertNotNull(result); + assertArrayEquals(dataFieldValue, (byte[])result.getValue()); + + collection.deleteCollectionAsync().block(); + } + + + + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123", null), + Arguments.of("boolean_true", Boolean.class, true, null), + Arguments.of("boolean_false", Boolean.class, false, null), + Arguments.of("byte", Byte.class, (byte) 127, null), + Arguments.of("short", Short.class, (short) 3, null), + Arguments.of("integer", Integer.class, 321, null), + Arguments.of("long", Long.class, 5L, null), + Arguments.of("float", Float.class, 3.14f, null), + Arguments.of("double", Double.class, 3.14159265358d, null), + Arguments.of("decimal", BigDecimal.class, new BigDecimal("12345.67"), null), + Arguments.of("timestamp", OffsetDateTime.class, OffsetDateTime.now(), null), + Arguments.of("uuid", UUID.class, UUID.randomUUID(), null), + Arguments.of("json", List.class, Arrays.asList("a", "s", "d"), String.class) + ); + } + +} From 0e044a08e52ecbb4bd694c2eabff7f501467f3b8 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Thu, 26 Jun 2025 18:59:06 +0200 Subject: [PATCH 35/64] Javadod and comments + map UUID to string and other small changes --- data/semantickernel-data-oracle/pom.xml | 30 +- .../jdbc/oracle/OracleDataTypesMapping.java | 58 ++- .../oracle/OracleVectorStoreFieldHelper.java | 206 +++------- .../OracleVectorStoreQueryProvider.java | 381 ++++++++++++------ .../oracle/OracleVectorStoreRecordMapper.java | 41 +- .../jdbc/oracle/ClassWithAllBoxedTypes.java | 6 + .../oracle/ClassWithAllPrimitiveTypes.java | 6 + .../jdbc/oracle/ClassWithAnnotatedTypes.java | 49 ++- .../data/jdbc/oracle/Hotel.java | 3 + ...CommonVectorStoreRecordCollectionTest.java | 6 + .../OracleVectorStoreAnnotatedTypeTest.java | 102 ++--- .../OracleVectorStoreDataTypeSearchTest.java | 11 +- .../oracle/OracleVectorStoreDataTypeTest.java | 6 + ...OracleVectorStoreRecordCollectionTest.java | 6 + 14 files changed, 482 insertions(+), 429 deletions(-) diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index 2a642271..9f7223f3 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -56,38 +56,10 @@ jackson-datatype-jsr310 2.18.0 - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - - org.apache.commons - commons-text - - - org.postgresql - postgresql - 42.7.4 - - - org.xerial - sqlite-jdbc - 3.47.0.0 - com.oracle.database.jdbc ojdbc11 @@ -98,6 +70,7 @@ ojdbc-provider-jackson-oson 1.0.4 + org.junit.jupiter junit-jupiter @@ -108,7 +81,6 @@ junit-jupiter-api test - org.testcontainers testcontainers diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index 0258cb5f..cfca9ecc 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -1,22 +1,74 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; /** - * Defines oracle database type constants for supported field types. + * Defines oracle database type constants for supported java types. */ public class OracleDataTypesMapping { - public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; + + /** + * Oracle database type used when strings are mapped to VARCHAR + */ + public static final String STRING_VARCHAR = "VARCHAR2(%s)"; + /** + * Oracle database type used when strings are mapped to CLOB + */ public static final String STRING_CLOB = "CLOB"; + /** + * Oracle database type used to map booleans + */ public static final String BOOLEAN = "BOOLEAN"; + /** + * Oracle database type used to map bytes + */ public static final String BYTE = "NUMBER(3)"; + /** + * Oracle database type used to map byte arrays + */ public static final String BYTE_ARRAY = "RAW(2000)"; + /** + * Oracle database type used to map shorts + */ public static final String SHORT = "NUMBER(5)"; + /** + * Oracle database type used to map ints + */ public static final String INTEGER = "NUMBER(10)"; + /** + * Oracle database type used to map longs + */ public static final String LONG = "NUMBER(19)"; + /** + * Oracle database type used to map float + */ public static final String FLOAT = "BINARY_FLOAT"; + /** + * Oracle database type used to map double + */ public static final String DOUBLE = "BINARY_DOUBLE"; + /** + * Oracle database type used to map BigDecimal + */ public static final String DECIMAL = "NUMBER"; + /** + * Oracle database type used to map offset date time + */ public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; - public static final String UUID = "RAW(16)"; + /** + * Oracle database type used to map UUID + */ + public static final String UUID = "VARCHAR2(36)"; + /** + * Oracle database type used to map lists + */ public static final String JSON = "JSON"; + /** + * Oracle database type used to map vectors (the parameter is the dimension of the vector) + */ public static final String VECTOR_FLOAT = "VECTOR(%s, FLOAT32)"; } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index dbc77178..c177131f 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; @@ -17,10 +23,15 @@ import java.util.stream.Collectors; /** - * Helper class for field operations. + * Helper class for field operations. Handles mapping between field java types to DB types and + * generating SQL statement to create field indexes. */ -public class OracleVectorStoreFieldHelper { - private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); +class OracleVectorStoreFieldHelper { + + /** + * The logger + */ + private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreFieldHelper.class.getName()); /** * Maps supported key java classes to Oracle database types @@ -28,13 +39,6 @@ public class OracleVectorStoreFieldHelper { private static final HashMap, String> supportedKeyTypes = new HashMap() { { put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); - put(short.class, OracleDataTypesMapping.SHORT); - put(Short.class, OracleDataTypesMapping.SHORT); - put(int.class, OracleDataTypesMapping.INTEGER); - put(Integer.class, OracleDataTypesMapping.INTEGER); - put(long.class, OracleDataTypesMapping.LONG); - put(Long.class, OracleDataTypesMapping.LONG); - put(UUID .class, OracleDataTypesMapping.UUID); } }; @@ -48,14 +52,6 @@ public class OracleVectorStoreFieldHelper { put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); -/* - put(byte[].class,"VECTOR(%s, INT8)"); - put(Byte[].class,"VECTOR(%s, INT8)"); - put(double[].class,"VECTOR(%s, FLOAT64)"); - put(Double[].class,"VECTOR(%s, FLOAT64)"); - put(boolean[].class,"VECTOR(%s, BINARY)"); - put(Boolean[].class,"VECTOR(%s, BINARY)"); - */ } }; @@ -84,26 +80,12 @@ public class OracleVectorStoreFieldHelper { put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); put(List.class, OracleDataTypesMapping.JSON); } - }; /** - * Maps vector type to OracleTypes. Only needed if types other than FLOAT_32 are supported. + * Suffix added to the effective column name to generate the index name for a vector column. */ - private static final Map, Integer> mapOracleTypeToVector = new HashMap() { - { - put(float[].class, OracleTypes.VECTOR_FLOAT32); - put(Float[].class, OracleTypes.VECTOR_FLOAT32); -/* - put(byte[].class, OracleTypes.VECTOR_INT8); - put(Byte[].class, OracleTypes.VECTOR_INT8); - put(Double[].class, OracleTypes.VECTOR_FLOAT64); - put(double[].class, OracleTypes.VECTOR_FLOAT64); - put(Boolean[].class, OracleTypes.VECTOR_BINARY); - put(boolean[].class, OracleTypes.VECTOR_BINARY); -*/ - } - }; + public static final String VECTOR_INDEX_SUFFIX = "_VECTOR_INDEX"; /** * Gets the mapping between the supported Java key types and the Oracle database type. @@ -121,12 +103,11 @@ public static HashMap, String> getSupportedKeyTypes() { */ public static Map, String> getSupportedDataTypes( StringTypeMapping stringTypeMapping, int defaultVarCharLength) { - - if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { - supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); - } else { - supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); - } + String stringType = stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR) + ? String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength) + : OracleDataTypesMapping.STRING_CLOB; + supportedDataTypes.put(String.class, stringType); + LOGGER.finest("Mapping String columns to " + stringType); return supportedDataTypes; } @@ -177,14 +158,14 @@ public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField */ public static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { - String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; + String dataFieldIndex = "CREATE MULTIVALUE INDEX IF NOT EXISTS %s ON %s t (t.%s.%s)"; return String.format(dataFieldIndex, collectionTableName + "_" + dataField.getEffectiveStorageName(), collectionTableName, dataField.getEffectiveStorageName(), getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); } else { - String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; + String dataFieldIndex = "CREATE INDEX IF NOT EXISTS %s ON %s (%s ASC)"; return String.format(dataFieldIndex, collectionTableName + "_" + dataField.getEffectiveStorageName(), collectionTableName, @@ -194,95 +175,28 @@ public static String createIndexForDataField(String collectionTableName, VectorS } /** - * Gets the function that allows to return the function that converts the JSON value to the - * data type. - * @param jdbcType The JDBC type. - * @return the function that allows to return the function that converts the JSON value to the - * data type. + * Returns vector columns names and types for CREATE TABLE statement + * @param fields list of vector record fields. + * @return comma separated list of columns and types for CREATE TABLE statement. */ - private static String getFunctionForType(String jdbcType) { - switch (jdbcType) { - case OracleDataTypesMapping.BOOLEAN: - return "boolean()"; - case OracleDataTypesMapping.BYTE: - case OracleDataTypesMapping.SHORT: - case OracleDataTypesMapping.INTEGER: - case OracleDataTypesMapping.LONG: - case OracleDataTypesMapping.FLOAT: - case OracleDataTypesMapping.DOUBLE: - case OracleDataTypesMapping.DECIMAL: - return "numberOnly()"; - case OracleDataTypesMapping.OFFSET_DATE_TIME: - return "timestamp()"; - default: - return "string()"; - } - } + public static String getVectorColumnNamesAndTypes(List fields) { + List columns = fields.stream() + .map(field -> field.getEffectiveStorageName() + " " + + OracleVectorStoreFieldHelper.getTypeForVectorField(field) + ).collect(Collectors.toList()); - /** - * Gets the type of the vector given the field definition. This method is not needed if only - * - * @param field the vector field definition. - * @return returns the type of vector for the given field type. - */ - public static String getTypeForVectorField(VectorStoreRecordVectorField field) { - String dimension = field.getDimensions() > 0 ? String.valueOf(field.getDimensions()) : "*"; - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); -/* Not needed since all types are FLOAT32 - if (field.getFieldSubType() != null) { - String vectorType; - switch (field.getFieldSubType().getName()) { - case "java.lang.Double": - vectorType = "FLOAT64"; - break; - case "java.lang.Byte": - vectorType = "INT8"; - break; - case "java.lang.Boolean": - vectorType = "BINARY"; - break; - default: - vectorType = "FLOAT32"; - } - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension, vectorType); - } else { - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); - } - */ + return String.join(", ", columns); } /** - * Gets the JDBC oracle of the vector field definition. - * @param field the vector field definition. - * @return the JDBC oracle type. + * Returns key column names and type for key column for CREATE TABLE statement + * @param field the key field. + * @return column name and type of the key field for CREATE TABLE statement. */ - public static int getOracleTypeForVectorField(VectorStoreRecordVectorField field) { - if (field.getFieldSubType() == null) { - Integer oracleType = mapOracleTypeToVector.get(field.getFieldType()); - if (oracleType != null) { - return oracleType.intValue(); - } else { - // field was declared as list with no subtype, assume FLOAT - return OracleTypes.VECTOR_FLOAT32; - } - - } else { - switch (field.getFieldSubType().getName()) { - case "java.lang.Double": - return OracleTypes.VECTOR_FLOAT64; - case "java.lang.Byte": - return OracleTypes.VECTOR_INT8; - case "java.lang.Boolean": - return OracleTypes.VECTOR_BINARY; - default: - return OracleTypes.VECTOR_FLOAT32; - } - } + public static String getKeyColumnNameAndType(VectorStoreRecordKeyField field) { + return field.getEffectiveStorageName() + " " + supportedKeyTypes.get(field.getFieldType()); } - public static boolean isUUID (VectorStoreRecordField field) { - return (field.getFieldType().getName() == "java.util.UUID"); - } /** * Generates the index name given the field name. by suffixing "_VECTOR_INDEX" to the field name. @@ -290,30 +204,44 @@ public static boolean isUUID (VectorStoreRecordField field) { * @return the index name. */ private static String getIndexName(String effectiveStorageName) { - return effectiveStorageName + "_VECTOR_INDEX"; + return effectiveStorageName + VECTOR_INDEX_SUFFIX; } /** - * Returns vector columns names and types for CREATE TABLE statement - * @param fields list of vector record fields. - * @return comma separated list of columns and types for CREATE TABLE statement. + * Gets the type of the vector given the field definition. This method is not needed if only + * + * @param field the vector field definition. + * @return returns the type of vector for the given field type. */ - public static String getVectorColumnNamesAndTypes(List fields) { - List columns = fields.stream() - .map(field -> field.getEffectiveStorageName() + " " + - OracleVectorStoreFieldHelper.getTypeForVectorField(field) - ).collect(Collectors.toList()); - - return String.join(", ", columns); + private static String getTypeForVectorField(VectorStoreRecordVectorField field) { + String dimension = field.getDimensions() > 0 ? String.valueOf(field.getDimensions()) : "*"; + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); } /** - * Returns key column names and type for key column for CREATE TABLE statement - * @param field the key field. - * @return column name and type of the key field for CREATE TABLE statement. + * Gets the function that allows to return the function that converts the JSON value to the + * data type. + * @param jdbcType The JDBC type. + * @return the function that allows to return the function that converts the JSON value to the + * data type. */ - public static String getKeyColumnNameAndType(VectorStoreRecordKeyField field) { - return field.getEffectiveStorageName() + " " + supportedKeyTypes.get(field.getFieldType()); + private static String getFunctionForType(String jdbcType) { + switch (jdbcType) { + case OracleDataTypesMapping.BOOLEAN: + return "boolean()"; + case OracleDataTypesMapping.BYTE: + case OracleDataTypesMapping.SHORT: + case OracleDataTypesMapping.INTEGER: + case OracleDataTypesMapping.LONG: + case OracleDataTypesMapping.FLOAT: + case OracleDataTypesMapping.DOUBLE: + case OracleDataTypesMapping.DECIMAL: + return "numberOnly()"; + case OracleDataTypesMapping.OFFSET_DATE_TIME: + return "timestamp()"; + default: + return "string()"; + } } } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index fa606e64..eba71db7 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -1,12 +1,16 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.util.ISO8601DateFormat; -import com.fasterxml.jackson.databind.util.StdDateFormat; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; @@ -28,31 +32,24 @@ import oracle.jdbc.OraclePreparedStatement; import oracle.jdbc.OracleStatement; import oracle.jdbc.OracleTypes; -import oracle.sql.TIMESTAMPLTZ; import oracle.sql.TIMESTAMPTZ; import javax.annotation.Nonnull; import javax.annotation.concurrent.GuardedBy; import javax.sql.DataSource; import java.math.BigDecimal; -import java.nio.ByteBuffer; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.SQLType; import java.sql.Statement; import java.sql.SQLException; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; -import java.time.LocalDateTime; import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -68,11 +65,15 @@ public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider // This could be common to all query providers private final ObjectMapper objectMapper; - private final Map, String> annotatedTypeMapping; - - private static final Object dbCreationLock = new Object(); + /** + * Lock used to ensure that only one thread can create a collection at a time. + */ + private static final ReentrantLock dbCreationLock = new ReentrantLock(); - private static final Logger logger = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); + /** + * The logger + */ + private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); public enum StringTypeMapping { /** @@ -86,8 +87,9 @@ public enum StringTypeMapping { } /** - * Constructor - * @param dataSource the datasiyrce + * Create an instance of OracleVectorStoreQueryProvider. + * + * @param dataSource the datasource * @param collectionsTable the collections table name * @param prefixForCollectionTables the prefix for the collection table name * @param defaultVarcharSize the size of VARCHAR columns @@ -100,8 +102,7 @@ private OracleVectorStoreQueryProvider( @Nonnull String prefixForCollectionTables, int defaultVarcharSize, @Nonnull StringTypeMapping stringTypeMapping, - ObjectMapper objectMapper, - Map, String> annotatedTypeMapping) { + ObjectMapper objectMapper) { super( dataSource, collectionsTable, @@ -111,17 +112,33 @@ private OracleVectorStoreQueryProvider( OracleVectorStoreFieldHelper.getSupportedVectorTypes()); this.collectionsTable = collectionsTable; this.objectMapper = objectMapper; + // The JavaTimeModule must be registered to handle OffsetDateTime. To make sure that it is + // registered enable the feature IGNORE_DUPLICATE_MODULE_REGISTRATIONS and register the + // module. + this.objectMapper.enable(MapperFeature.IGNORE_DUPLICATE_MODULE_REGISTRATIONS); this.objectMapper.registerModule(new JavaTimeModule()); - this.annotatedTypeMapping = annotatedTypeMapping; } + /** + *

+ * Creates a collection with the given name and record definition. + *

+ * A collection is represented as a table in an Oracle DB containing columns + * that match the record definition. The table name is the name of the collection + * prefixed by the provided collection prefix. If no prefix was provided the default + * prefix will be used. + *

+ * @param collectionName the name of the collection + * @param recordDefinition the record definition + */ @Override @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") @GuardedBy("dbCreationLock") public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { - synchronized (dbCreationLock) { + dbCreationLock.lock(); + try { List vectorFields = recordDefinition.getVectorFields(); String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" @@ -138,39 +155,47 @@ public void createCollection(String collectionName, String insertCollectionQuery = this.getInsertCollectionQuery(collectionsTable); try (Connection connection = dataSource.getConnection()) { - connection.createStatement().execute(formatQuery("DROP TABLE IF EXISTS %s", getCollectionTableName(collectionName))); + // set auto commit of, either all statements should be executed or none connection.setAutoCommit(false); try (Statement statement = connection.createStatement()) { // Create table - System.out.println(createStorageTable); statement.addBatch(createStorageTable); + LOGGER.finest("Creating collection " + collectionName + + " using statement: " + createStorageTable); - // Index filterable columns + // Index filterable data columns for (VectorStoreRecordDataField dataField : recordDefinition.getDataFields()) { if (dataField.isFilterable()) { String dataFieldIndex = OracleVectorStoreFieldHelper.createIndexForDataField( getCollectionTableName(collectionName), dataField, supportedDataTypes); - System.out.println(dataFieldIndex); statement.addBatch(dataFieldIndex); + LOGGER.finest("Creating index on column " + + dataField.getEffectiveStorageName() + " using the statement: " + + dataFieldIndex); } } - // Create indexed for vectorFields + // Create index for vectorFields for (VectorStoreRecordVectorField vectorField : vectorFields) { String createVectorIndex = OracleVectorStoreFieldHelper.getCreateVectorIndexStatement( vectorField, getCollectionTableName(collectionName)); if (createVectorIndex != null) { - System.out.println(createVectorIndex); statement.addBatch(createVectorIndex); + LOGGER.finest("Creating index on vector column " + + vectorField.getEffectiveStorageName() + " using the statement: " + + createVectorIndex); + } } statement.executeBatch(); + // Insert the collection to the store (collections table) using MERGE statement try (PreparedStatement insert = connection.prepareStatement( insertCollectionQuery)) { - System.out.println(insertCollectionQuery); insert.setString(1, collectionName); insert.execute(); + LOGGER.finest("Inserting collection to store using statement: " + + insertCollectionQuery); } connection.commit(); @@ -181,161 +206,216 @@ public void createCollection(String collectionName, } catch (SQLException e) { throw new SKException("Failed to create collection", e); } + } finally { + dbCreationLock.unlock(); } } + /** + *

+ * Inserts or updates record of a collection given the collection name, the records, the record + * definition and the upsert options. + *

+ * @Note At the moment {@link UpsertRecordOptions} is an empty class. No options are available. + *

+ * + * @param collectionName the collection name + * @param records the records to update or insert + * @param recordDefinition the record definition + * @param options the options + */ @Override - protected String getInsertCollectionQuery(String collectionsTable) { - return formatQuery( - "MERGE INTO %s existing "+ - "USING (SELECT ? AS collectionId FROM DUAL) new ON (existing.collectionId = new.collectionId) " + - "WHEN NOT MATCHED THEN INSERT (existing.collectionId) VALUES (new.collectionId)", - collectionsTable); - } - - @Override - public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + public void upsertRecords(String collectionName, + List records, + VectorStoreRecordDefinition recordDefinition, + UpsertRecordOptions options) { final String NEW_VALUE = "new"; final String EXISTING_VALUE = "existing"; + // generate the comma separated list of new fields + // Ex.: new.field1, new.field2 ... new.fieldn String insertNewFieldList = recordDefinition.getAllFields().stream() .map(f -> NEW_VALUE + "." + f.getEffectiveStorageName()) .collect(Collectors.joining(", ")); + // generate the comma separated list of existing fields + // Ex.: existing.field1, existing.field2 ... existing.fieldn String insertExistingFieldList = recordDefinition.getAllFields().stream() .map(f -> EXISTING_VALUE + "." + f.getEffectiveStorageName()) .collect(Collectors.joining(", ")); + // generate the comma separated list for setting new values on fields + // Ex.: new.field1 = existing.field1, new.field2 = existing.field2 ... new.fieldn = existing.fieldn String updateFieldList = recordDefinition.getAllFields().stream() .filter(f -> f != recordDefinition.getKeyField()) .map(f -> EXISTING_VALUE + "." + f.getEffectiveStorageName() + " = " + NEW_VALUE + "." + f.getEffectiveStorageName()) .collect(Collectors.joining(", ")); - String namedWildcard = recordDefinition.getAllFields().stream().map(f -> "? " + f.getEffectiveStorageName()) + // generate the comma separated list of placeholders "?" for each field + // Ex.: ? field1, ? field2 ... ? fieldn + String namedPlaceholders = recordDefinition.getAllFields().stream().map(f -> "? " + f.getEffectiveStorageName()) .collect(Collectors.joining(", ")); - String upsertQuery = formatQuery("MERGE INTO %s existing "+ + // Generate the MERGE statement to perform the upsert. + String upsertStatement = formatQuery("MERGE INTO %s existing "+ "USING (SELECT %s FROM DUAL) new ON (existing.%s = new.%s) " + "WHEN MATCHED THEN UPDATE SET %s " + "WHEN NOT MATCHED THEN INSERT (%s) VALUES (%s)", getCollectionTableName(collectionName), - namedWildcard, + namedPlaceholders, getKeyColumnName(recordDefinition.getKeyField()), getKeyColumnName(recordDefinition.getKeyField()), updateFieldList, insertExistingFieldList, insertNewFieldList); - System.out.println(upsertQuery); + LOGGER.finest("Generated upsert statement: " + upsertStatement); try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(upsertQuery)) { + PreparedStatement statement = connection.prepareStatement(upsertStatement)) { + // Loop through records, set values and add values to batch for (Object record : records) { setUpsertStatementValues(statement, record, recordDefinition.getAllFields()); statement.addBatch(); } + // Execute the upsert statement statement.executeBatch(); } catch (SQLException e) { throw new SKException("Failed to upsert records", e); } } - private void setUpsertStatementValues(PreparedStatement statement, Object record, + /** + * Generates the MERGE statement to add the given collection to the store. + * + * @param collectionsTable the name of the DB table containing all collections. + * @return a SQL statement that inserts a collection to the store if it does not exist. + */ + @Override + protected String getInsertCollectionQuery(String collectionsTable) { + return formatQuery( + "MERGE INTO %s existing "+ + "USING (SELECT ? AS collectionId FROM DUAL) new ON (existing.collectionId = new.collectionId) " + + "WHEN NOT MATCHED THEN INSERT (existing.collectionId) VALUES (new.collectionId)", + collectionsTable); + } + + /** + * The {@link OracleVectorStoreQueryProvider#upsertRecords(String, List, VectorStoreRecordDefinition, UpsertRecordOptions)} + * method adds a placeholder for each field. This method sets the value of each field on the + * MERGE statement with the value of the record. The placeholder and values are set in the order + * of the fields in the list. + * + * @param upsertStatement the MERGE statement + * @param record the record containing the values + * @param fields the list of fields. + */ + private void setUpsertStatementValues(PreparedStatement upsertStatement, Object record, List fields) { - objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - String datePattern = "yyyy-MM-dd HH:mm:ss.SSSZ"; - SimpleDateFormat df = new SimpleDateFormat(datePattern); - objectMapper.setDateFormat(df); + + // use the object mapper to convert the record to an equivalent tree mode JsonNode value, + // this allows to retrieve the values using the effective storage name of the fields and + // avoids the use of introspection. JsonNode jsonNode = objectMapper.valueToTree(record); for (int i = 0; i < fields.size(); ++i) { VectorStoreRecordField field = fields.get(i); try { + JsonNode valueNode = jsonNode.get(field.getEffectiveStorageName()); + // Some field types require special treatment to convert the java type to the + // DB type if (field instanceof VectorStoreRecordVectorField) { - // Convert the vector field to a string + // If the vector field is not set as a string convert to an array of doubles + // and set the value if (!field.getFieldType().equals(String.class)) { double[] values = (valueNode == null || valueNode.isNull()) ? null : StreamSupport.stream(( (ArrayNode)valueNode).spliterator(), false) .mapToDouble(d -> d.asDouble()).toArray(); - statement.setObject(i + 1, values, - OracleVectorStoreFieldHelper.getOracleTypeForVectorField((VectorStoreRecordVectorField)field)); - System.out.println("Set values: " + values); + upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); continue; } } else if (field instanceof VectorStoreRecordDataField) { - // Convert List field to a string + // Lists are stored as JSON objects, write the list as a JSON string representation + // of the list. if (field.getFieldType().equals(List.class)) { - statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); - System.out.println( - "Set values: " + objectMapper.writeValueAsString(valueNode)); + upsertStatement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); continue; } - if (OracleVectorStoreFieldHelper.isUUID(field)) { - if (valueNode == null || valueNode.isNull()) { - statement.setNull(i + 1, OracleTypes.RAW); - } else { - UUID uuid = UUID.fromString(valueNode.textValue()); - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - statement.setBytes(i + 1, bb.array()); - System.out.println("Set values: " + objectMapper.convertValue(valueNode, - field.getFieldType())); - } + // Convert UUID to string before setting the value. + if (field.getFieldType().equals(UUID.class)) { + upsertStatement.setObject(i + 1, valueNode.isNull() ? null : valueNode.asText()); continue; } + // Convert OffsetDateTime to TIMESTAMPTZ before setting the value. if (field.getFieldType().equals(OffsetDateTime.class)) { if (valueNode == null || valueNode.isNull()) { - statement.setNull(i + 1, OracleTypes.TIMESTAMPTZ); + upsertStatement.setNull(i + 1, OracleTypes.TIMESTAMPTZ); } else { - OffsetDateTime offsetDateTime = OffsetDateTime.parse( - valueNode.asText()); - ((OraclePreparedStatement) statement).setTIMESTAMPTZ(i + 1, + OffsetDateTime offsetDateTime = (OffsetDateTime) objectMapper.convertValue(valueNode, field.getFieldType()); + ((OraclePreparedStatement) upsertStatement).setTIMESTAMPTZ(i + 1, TIMESTAMPTZ.of(offsetDateTime)); - System.out.println("Set values: " + objectMapper.convertValue(valueNode, - field.getFieldType())); } continue; } } - statement.setObject(i + 1, + // For all other field type use setObject with the field value + upsertStatement.setObject(i + 1, objectMapper.convertValue(valueNode,field.getFieldType())); - System.out.println("Set values: " + objectMapper.convertValue(valueNode, field.getFieldType())); } catch (SQLException | JsonProcessingException e) { throw new RuntimeException(e); } } } + /** + *

+ * Executes a vector search query, using the search options and returns the results. The results + * are mapped to the specified record type using the provided mapper. The query is executed + * against the specified collection. + *

+ * + *

+ * @param collectionName the collection name + * @param vector the vector to search with + * @param options the search options + * @param recordDefinition the record definition + * @param mapper the mapper, responsible for mapping the result set to the record + * type. + * @return the search results + * @param the record type + */ @Override public VectorSearchResults search(String collectionName, List vector, VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper) { - VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() - .get(0); + // Gets the search vector field and its distance function. If not vector field was provided, + // use the first one VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null - ? firstVectorField + ? recordDefinition.getVectorFields().get(0) : (VectorStoreRecordVectorField) recordDefinition .getField(options.getVectorFieldName()); - DistanceFunction distanceFunction = vectorField.getDistanceFunction(); - - List fields; - if (options.isIncludeVectors()) { - fields = recordDefinition.getAllFields(); - } else { - fields = recordDefinition.getNonVectorFields(); + DistanceFunction distanceFunction = vectorField == null ? null : vectorField.getDistanceFunction(); + if (options.getVectorFieldName() != null && vectorField == null) { + throw new SKException(""); } + // get list of fields that should be returned by the query + List fields = (options.isIncludeVectors()) + ? recordDefinition.getAllFields() + : recordDefinition.getNonVectorFields(); + + // get search filters and get the list of parameters for the filters String filter = getFilter(options.getVectorSearchFilter(), recordDefinition); List parameters = getFilterParameters(options.getVectorSearchFilter()); + // generate SQL statement String selectQuery = "SELECT " + (vector == null ? "0 as distance, " : formatQuery("VECTOR_DISTANCE(%s, ?, %s) distance, ", vectorField.getEffectiveStorageName(), toOracleDistanceFunction(distanceFunction))) @@ -345,19 +425,20 @@ public VectorSearchResults search(String collectionName, List 0 ? " OFFSET " + options.getSkip() + " ROWS" : "") + (options.getTop() > 0 ? " FETCH " + (options.getSkip() > 0 ? "NEXT " : "FIRST ") + options.getTop() + " ROWS ONLY" : ""); + LOGGER.finest("Search using statement: " + selectQuery); - System.out.println(selectQuery); + // Execute the statement List> records = new ArrayList<>(); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(selectQuery)) { // set parameters from filters int parameterIndex = 1; + // if a vector was provided for similarity search set the value of the vector if (vector != null) { - System.out.println("Set vector parameter with index " + parameterIndex + " to: " - + objectMapper.writeValueAsString(vector)); statement.setString(parameterIndex++, objectMapper.writeValueAsString(vector)); } + // set all parameters. for (Object parameter : parameters) { if (parameter != null) { setSearchParameter(statement, parameterIndex++, parameter.getClass(), parameter); @@ -371,18 +452,19 @@ public VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List(mapper.mapStorageModelToRecord(rs, getRecordOptions), score)); } } } catch (SQLException | JsonProcessingException e) { - logger.info(e.getMessage()); throw new SKException("Search failed", e); } return new VectorSearchResults<>(records); } + /** + * Sets the parameter value + * @param statement the statement + * @param index the parameter index + * @param type the parameter type + * @param value the value + */ private void setSearchParameter(PreparedStatement statement, int index, Class type, Object value) { try { + // Use JSON string to set lists if (List.class.equals(type)) { statement.setObject(index, objectMapper.writeValueAsString(value)); System.out.println( "Set values: " + objectMapper.writeValueAsString(value)); return; } + // convert UUID to string if (UUID.class.equals(type)) { - if (value == null) { - statement.setNull(index, OracleTypes.RAW); - } else { - UUID uuid = (UUID)value; - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - statement.setBytes(index, bb.array()); - System.out.println("Set values: " + uuid); - } + statement.setString(index, value.toString()); return; } + // convert OffsetDateType to TIMESTAMPTZ if (OffsetDateTime.class.equals(type)) { if (value == null) { statement.setNull(index, OracleTypes.TIMESTAMPTZ); @@ -435,6 +518,7 @@ private void setSearchParameter(PreparedStatement statement, int index, Class } return; } + // use setBigDecimal to set BigDecimal value if (BigDecimal.class.equals(type)) { if (value == null) { statement.setNull(index, OracleTypes.DECIMAL); @@ -446,7 +530,8 @@ private void setSearchParameter(PreparedStatement statement, int index, Class } return; } - System.out.println("Set parameter " + index + " to: " + value); + + // for all other types set object with the given value statement.setObject(index, value); } catch (Exception ex) { @@ -455,6 +540,13 @@ private void setSearchParameter(PreparedStatement statement, int index, Class } + /** + * Defines the type that will be used to retrieve data from a given database table column. + * @param columnIndex the index of the column + * @param statement the statement + * @param fieldType the java field type + * @throws SQLException if an error occurs while defining the column type + */ private void defineDataColumnType(int columnIndex, OracleStatement statement, Class fieldType) throws SQLException { // swich between supported classes and define the column type on the statement switch (supportedDataTypes.get(fieldType)) { @@ -491,9 +583,6 @@ private void defineDataColumnType(int columnIndex, OracleStatement statement, Cl case OracleDataTypesMapping.JSON: statement.defineColumnType(columnIndex, OracleTypes.JSON, Integer.MAX_VALUE); break; - case OracleDataTypesMapping.UUID: - statement.defineColumnType(columnIndex, OracleTypes.RAW); - break; case OracleDataTypesMapping.BYTE_ARRAY: statement.defineColumnType(columnIndex, OracleTypes.RAW); default: @@ -501,7 +590,11 @@ private void defineDataColumnType(int columnIndex, OracleStatement statement, Cl } } - + /** + * Converts a {@link DistanceFunction} to the equivalent Oracle distance function. + * @param distanceFunction the distance function + * @return the Oracle distance function + */ private String toOracleDistanceFunction(DistanceFunction distanceFunction) { switch (distanceFunction) { case DOT_PRODUCT: @@ -525,6 +618,7 @@ private String toOracleDistanceFunction(DistanceFunction distanceFunction) { */ @Override public List getFilterParameters(VectorSearchFilter filter) { + // TODO: this method should be protected, not public if (filter == null || filter.getFilterClauses().isEmpty()) { return Collections.emptyList(); @@ -544,6 +638,11 @@ public List getFilterParameters(VectorSearchFilter filter) { }).collect(Collectors.toList()); } + /** + * Gets the filter clause for an equal to filter + * @param filterClause The equal to filter clause to get the filter string for. + * @return the filter clause + */ @Override public String getEqualToFilter(EqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider @@ -557,6 +656,11 @@ public String getEqualToFilter(EqualToFilterClause filterClause) { } } + /** + * Gets the filter clause for an any tag equal to filter + * @param filterClause The any tag equal to filter clause to get the filter string for. + * @return the filter clause + */ @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider @@ -566,6 +670,13 @@ public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { fieldName, fieldName, fieldName); } + /** + * Gets the mapper used to map a ResultSet to records + * @param recordClass the record class + * @param vectorStoreRecordDefinition the record definition + * @return the vector store record mapper + * @param the type of the records + */ @Override public VectorStoreRecordMapper getVectorStoreRecordMapper( Class recordClass, @@ -574,25 +685,52 @@ public VectorStoreRecordMapper getVectorStoreRecordM .withRecordClass(recordClass) .withVectorStoreRecordDefinition(vectorStoreRecordDefinition) .withSupportedDataTypesMapping(getSupportedDataTypes()) - .withAnnotatedTypeMapping(annotatedTypeMapping) .build(); } + /** + * Gets a builder that allows to build an OracleVectorStoreQueryProvider + * @return the builder + */ public static Builder builder() { return new Builder(); } + /** + * OracleVectorStoreQueryProvider builder. + */ public static class Builder extends JDBCVectorStoreQueryProvider.Builder { + /** + * The data source + */ private DataSource dataSource; + + /** + * The collections table + */ private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + + /** + * The prefix for collection table names + */ private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + + /** + * The object mapper + */ private ObjectMapper objectMapper = new ObjectMapper(); + + /** + * The string type mapping choice + */ private StringTypeMapping stringTypeMapping = StringTypeMapping.USE_VARCHAR; - private int defaultVarcharSize = 2000; - private Map, String> annotatedTypeMapping = null; + /** + * The size of varchar columns + */ + private int defaultVarcharSize = 2000; @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -621,17 +759,17 @@ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { return this; } + /** + * Sets the object mapper used to map records to and from results + * @param objectMapper the object mapper + * @return the builder + */ public Builder withObjectMapper( ObjectMapper objectMapper) { this.objectMapper = objectMapper; return this; } - public Builder withAnnotatedTypeMapping(Map, String> annotatedTypeMapping) { - this.annotatedTypeMapping = annotatedTypeMapping; - return this; - } - /** * Sets the desired String type mapping. * @param stringTypeMapping the desired String type mapping. The default value is @@ -644,8 +782,8 @@ public Builder withStringTypeMapping (StringTypeMapping stringTypeMapping) { } /** - * Sets the default size of the VARHCHAR2 fields. - * @param defaultVarcharSize the default size of the VARHCHAR2 fields. By default, the size + * Sets the default size of the VARHCHAR fields. + * @param defaultVarcharSize the default size of the VARHCHAR fields. By default, the size * is 2000. * @return then builder */ @@ -654,11 +792,14 @@ public Builder withDefaultVarcharSize (int defaultVarcharSize) { return this; } + /** + * Builds and Oracle vector store query provider. + * @return the query provider + */ @Override public OracleVectorStoreQueryProvider build() { return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, - prefixForCollectionTables, defaultVarcharSize, stringTypeMapping, objectMapper, - annotatedTypeMapping); + prefixForCollectionTables, defaultVarcharSize, stringTypeMapping, objectMapper); } } } \ No newline at end of file diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index e6e7bf0a..fd4f9dbf 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -1,11 +1,14 @@ -// Copyright (c) Microsoft. All rights reserved. +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -17,7 +20,6 @@ import oracle.jdbc.OracleResultSet; import oracle.jdbc.provider.oson.OsonModule; import oracle.sql.TIMESTAMPTZ; -import java.nio.ByteBuffer; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Map; @@ -150,14 +152,6 @@ public OracleVectorStoreRecordMapper build() { for (VectorStoreRecordField field : vectorStoreRecordDefinition.getNonVectorFields()) { Class fieldType = field.getFieldType(); - boolean isAnnotated = false; - try { - isAnnotated = recordClass.getDeclaredField(field.getName()) - .isAnnotationPresent(JsonTypeInfo.class); - } catch (NoSuchFieldException e) { - // ignore exception, assume the field is not annotated - } - Object value; switch (supportedDataTypesMapping.get(fieldType)) { case OracleDataTypesMapping.STRING_CLOB: @@ -195,15 +189,8 @@ public OracleVectorStoreRecordMapper build() { value = resultSet.getBytes(field.getEffectiveStorageName()); break; case OracleDataTypesMapping.UUID: - byte[] bytes = resultSet.getBytes(field.getEffectiveStorageName()); - if (bytes != null) { - ByteBuffer bb = ByteBuffer.wrap(bytes); - long firstLong = bb.getLong(); - long secondLong = bb.getLong(); - value = new UUID(firstLong, secondLong); - } else { - value = null; - } + String uuidValue = resultSet.getString(field.getEffectiveStorageName()); + value = uuidValue == null ? null : UUID.fromString(uuidValue); break; case OracleDataTypesMapping.JSON: value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); @@ -220,18 +207,6 @@ public OracleVectorStoreRecordMapper build() { JsonNode genericNode = objectMapper.valueToTree(value); objectNode.set(field.getEffectiveStorageName(), genericNode); - if (isAnnotated) { - if (annotatedTypeMapping != null && annotatedTypeMapping.containsKey(field.getFieldType())) { - objectNode.set(field.getEffectiveStorageName() + "_type", - TextNode.valueOf( - annotatedTypeMapping.get(field.getFieldType()))); - } else { - objectNode.set(field.getEffectiveStorageName() + "_type", - TextNode.valueOf( - field.getFieldType().getName())); - - } - } } if (options != null && options.isIncludeVectors()) { for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java index 9f93e03d..307c974d 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java index 6e91485d..0ca52d7b 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java index 318911d8..5964cb3c 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java @@ -1,41 +1,45 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonTypeInfo.As; -import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; -import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; -import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.List; import java.util.UUID; -@JsonIgnoreProperties(ignoreUnknown = true) public class ClassWithAnnotatedTypes { private final String id; + @JsonProperty("value_type") + private final String valueType; + + @JsonProperty("value_field") @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = As.EXTERNAL_PROPERTY , property = "value_type") @JsonSubTypes({ - @JsonSubTypes.Type(value = String.class, name="java.lang.String"), - @JsonSubTypes.Type(value = Boolean.class, name="java.lang.Boolean"), - @JsonSubTypes.Type(value = Byte.class, name="java.lang.Byte"), - @JsonSubTypes.Type(value = Short.class, name="java.lang.Short"), - @JsonSubTypes.Type(value = Integer.class, name="java.lang.Integer"), - @JsonSubTypes.Type(value = Long.class, name="java.lang.Long"), - @JsonSubTypes.Type(value = Float.class, name="java.lang.Float"), - @JsonSubTypes.Type(value = Double.class, name="java.lang.Double"), - @JsonSubTypes.Type(value = BigDecimal.class, name="java.math.BigDecimal"), - @JsonSubTypes.Type(value = OffsetDateTime.class, name="java.time.OffsetDateTime"), - @JsonSubTypes.Type(value = UUID.class, name="java.util.UUID"), + @JsonSubTypes.Type(value = String.class, name="string"), + @JsonSubTypes.Type(value = Boolean.class, name="boolean"), + @JsonSubTypes.Type(value = Byte.class, name="byte"), + @JsonSubTypes.Type(value = Short.class, name="short"), + @JsonSubTypes.Type(value = Integer.class, name="integer"), + @JsonSubTypes.Type(value = Long.class, name="long"), + @JsonSubTypes.Type(value = Float.class, name="float"), + @JsonSubTypes.Type(value = Double.class, name="double"), + @JsonSubTypes.Type(value = BigDecimal.class, name="decimal"), + @JsonSubTypes.Type(value = OffsetDateTime.class, name="timestamp"), + @JsonSubTypes.Type(value = UUID.class, name="uuid"), @JsonSubTypes.Type(value = byte[].class, name="byte_array"), - @JsonSubTypes.Type(value = List.class, name="listOfStrings") + @JsonSubTypes.Type(value = List.class, name="json") }) private Object value; @@ -43,10 +47,11 @@ public class ClassWithAnnotatedTypes { public ClassWithAnnotatedTypes() { - this(null, null, null); + this(null, null, null, null); }; - public ClassWithAnnotatedTypes(String id, Object value, Float[] vectorValue) { + public ClassWithAnnotatedTypes(String id, String valueType, Object value, Float[] vectorValue) { this.id = id; + this.valueType = valueType; this.value = value; this.vectorValue = vectorValue; } @@ -55,6 +60,8 @@ public String getId() { return id; } + public String getValueType() { return valueType; } + public Object getValue() { return value; } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index d1c848a6..dc736ba3 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -1,4 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. + +// Should we change the copyright from the file we copied? Or should we +// just change the test to use a class that we create? package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java index f8656c3a..7549188b 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import oracle.jdbc.OracleConnection; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java index 9650652e..42036826 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; @@ -41,18 +47,26 @@ void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFiel if (fieldSubType != null) { dataField = VectorStoreRecordDataField.builder() .withName("value") - .withStorageName("value") + .withStorageName("value_field") .withFieldType(dataFieldType, fieldSubType) .isFilterable(true) .build(); } else { dataField = VectorStoreRecordDataField.builder() .withName("value") - .withStorageName("value") + .withStorageName("value_field") .withFieldType(dataFieldType) .isFilterable(true) .build(); } + VectorStoreRecordDataField dataTypeField; + dataTypeField = VectorStoreRecordDataField.builder() + .withName("valueType") + .withStorageName("value_type") + .withFieldType(String.class) + .isFilterable(false) + .build(); + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() .withName("vectorValue") @@ -64,14 +78,11 @@ void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFiel .build(); VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( - Arrays.asList(keyField, dataField, dummyVector) + Arrays.asList(keyField, dataTypeField, dataField, dummyVector) ); OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() .withDataSource(DATA_SOURCE) - .withAnnotatedTypeMapping(new HashMap() {{ - put(List.class, "listOfStrings"); - }}) .build(); JDBCVectorStore vectorStore = JDBCVectorStore.builder() @@ -94,7 +105,7 @@ JDBCVectorStoreRecordCollectionOptions. builder() String key = "testid"; ClassWithAnnotatedTypes record = - new ClassWithAnnotatedTypes(key, dataFieldValue, new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }); + new ClassWithAnnotatedTypes(key, dataFieldName, dataFieldValue, new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }); collection.upsertAsync(record, null).block(); @@ -102,6 +113,8 @@ JDBCVectorStoreRecordCollectionOptions. builder() assertNotNull(result); if (record.getValue().getClass().equals(OffsetDateTime.class)) { assertTrue(((OffsetDateTime)dataFieldValue).isEqual((OffsetDateTime)record.getValue())); + } else if (dataFieldName == "byte_array") { + assertArrayEquals((byte[]) dataFieldValue, (byte[])record.getValue()); } else { assertEquals(dataFieldValue, result.getValue()); } @@ -109,81 +122,11 @@ JDBCVectorStoreRecordCollectionOptions. builder() collection.deleteCollectionAsync().block(); } - @Test - void testByteArrayType() { - VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() - .withName("id") - .withStorageName("id") - .withFieldType(String.class) - .build(); - - VectorStoreRecordDataField dataField; - dataField = VectorStoreRecordDataField.builder() - .withName("value") - .withStorageName("value") - .withFieldType(byte[].class) - .isFilterable(true) - .build(); - - VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() - .withName("vectorValue") - .withStorageName("vectorValue") - .withFieldType(Float[].class) - .withDimensions(8) - .withDistanceFunction(DistanceFunction.COSINE_DISTANCE) - .withIndexKind(IndexKind.IVFFLAT) - .build(); - - VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( - Arrays.asList(keyField, dataField, dummyVector) - ); - - OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() - .withDataSource(DATA_SOURCE) - .withAnnotatedTypeMapping(new HashMap() {{ - put(byte[].class, "byte_array"); - }}) - .build(); - - JDBCVectorStore vectorStore = JDBCVectorStore.builder() - .withDataSource(DATA_SOURCE) - .withOptions(JDBCVectorStoreOptions.builder() - .withQueryProvider(queryProvider) - .build()) - .build(); - - String collectionName = "test_datatype_bytearray"; - - VectorStoreRecordCollection collection = - vectorStore.getCollection(collectionName, - JDBCVectorStoreRecordCollectionOptions. builder() - .withRecordClass(ClassWithAnnotatedTypes.class) - .withRecordDefinition(definition).build()); - - collection.createCollectionAsync().block(); - - String key = "testid"; - - byte[] dataFieldValue = new byte[] {1, 2, 3}; - ClassWithAnnotatedTypes record = - new ClassWithAnnotatedTypes(key, dataFieldValue, new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }); - - collection.upsertAsync(record, null).block(); - - ClassWithAnnotatedTypes result = collection.getAsync(key, null).block(); - assertNotNull(result); - assertArrayEquals(dataFieldValue, (byte[])result.getValue()); - - collection.deleteCollectionAsync().block(); - } - - - private static Stream supportedDataTypes() { return Stream.of( Arguments.of("string", String.class, "asd123", null), - Arguments.of("boolean_true", Boolean.class, true, null), - Arguments.of("boolean_false", Boolean.class, false, null), + Arguments.of("boolean", Boolean.class, true, null), + Arguments.of("boolean", Boolean.class, false, null), Arguments.of("byte", Byte.class, (byte) 127, null), Arguments.of("short", Short.class, (short) 3, null), Arguments.of("integer", Integer.class, 321, null), @@ -193,6 +136,7 @@ private static Stream supportedDataTypes() { Arguments.of("decimal", BigDecimal.class, new BigDecimal("12345.67"), null), Arguments.of("timestamp", OffsetDateTime.class, OffsetDateTime.now(), null), Arguments.of("uuid", UUID.class, UUID.randomUUID(), null), + Arguments.of("byte_array", byte[].class, new byte[] {1, 2, 3}, String.class), Arguments.of("json", List.class, Arrays.asList("a", "s", "d"), String.class) ); } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java index d99234ab..f81176b3 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; @@ -119,11 +125,6 @@ void testDataTypesSearch (ClassWithAllBoxedTypes record) { assertEquals(1, results.getTotalCount()); assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); - System.out.println(record.getDecimalValue()); - System.out.println(record.getDecimalValue().doubleValue()); - System.out.println(results.getResults().get(0).getRecord().getDecimalValue()); - System.out.println(results.getResults().get(0).getRecord().getDecimalValue().doubleValue()); - // decimal results = collection.searchAsync( null, diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java index 5c575950..cc8d2462 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index b3eb403c..1a7a328a 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonCreator; From abd6d42683bf8524353642b3746674686b3c8df4 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Thu, 26 Jun 2025 20:04:22 +0200 Subject: [PATCH 36/64] Fixed merge issues --- .../jdbc/oracle/OracleDataTypesMapping.java | 23 -- .../oracle/OracleVectorStoreFieldHelper.java | 166 ------------- .../oracle/OracleVectorStoreRecordMapper.java | 11 - .../OracleVectorStoreDataTypeSearchTest.java | 11 - ...OracleVectorStoreRecordCollectionTest.java | 229 ------------------ 5 files changed, 440 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index 17ae9b2d..cfca9ecc 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -1,4 +1,3 @@ -<<<<<<< add-oracle-store /* ** Semantic Kernel Oracle connector version 1.0. ** @@ -71,27 +70,5 @@ public class OracleDataTypesMapping { /** * Oracle database type used to map vectors (the parameter is the dimension of the vector) */ -======= -package com.microsoft.semantickernel.data.jdbc.oracle; - -/** - * Defines oracle database type constants for supported field types. - */ -public class OracleDataTypesMapping { - public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; - public static final String STRING_CLOB = "CLOB"; - public static final String BOOLEAN = "BOOLEAN"; - public static final String BYTE = "NUMBER(3)"; - public static final String BYTE_ARRAY = "RAW(2000)"; - public static final String SHORT = "NUMBER(5)"; - public static final String INTEGER = "NUMBER(10)"; - public static final String LONG = "NUMBER(19)"; - public static final String FLOAT = "BINARY_FLOAT"; - public static final String DOUBLE = "BINARY_DOUBLE"; - public static final String DECIMAL = "NUMBER"; - public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; - public static final String UUID = "RAW(16)"; - public static final String JSON = "JSON"; ->>>>>>> main public static final String VECTOR_FLOAT = "VECTOR(%s, FLOAT32)"; } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index 359f8f50..c177131f 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -1,20 +1,14 @@ -<<<<<<< add-oracle-store /* ** Semantic Kernel Oracle connector version 1.0. ** ** Copyright (c) 2025 Oracle and/or its affiliates. ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ */ -======= ->>>>>>> main package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; -<<<<<<< add-oracle-store import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; -======= ->>>>>>> main import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import oracle.jdbc.OracleTypes; @@ -29,7 +23,6 @@ import java.util.stream.Collectors; /** -<<<<<<< add-oracle-store * Helper class for field operations. Handles mapping between field java types to DB types and * generating SQL statement to create field indexes. */ @@ -39,12 +32,6 @@ class OracleVectorStoreFieldHelper { * The logger */ private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreFieldHelper.class.getName()); -======= - * Helper class for field operations. - */ -public class OracleVectorStoreFieldHelper { - private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); ->>>>>>> main /** * Maps supported key java classes to Oracle database types @@ -52,16 +39,6 @@ public class OracleVectorStoreFieldHelper { private static final HashMap, String> supportedKeyTypes = new HashMap() { { put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); -<<<<<<< add-oracle-store -======= - put(short.class, OracleDataTypesMapping.SHORT); - put(Short.class, OracleDataTypesMapping.SHORT); - put(int.class, OracleDataTypesMapping.INTEGER); - put(Integer.class, OracleDataTypesMapping.INTEGER); - put(long.class, OracleDataTypesMapping.LONG); - put(Long.class, OracleDataTypesMapping.LONG); - put(UUID .class, OracleDataTypesMapping.UUID); ->>>>>>> main } }; @@ -75,17 +52,6 @@ public class OracleVectorStoreFieldHelper { put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); -<<<<<<< add-oracle-store -======= -/* - put(byte[].class,"VECTOR(%s, INT8)"); - put(Byte[].class,"VECTOR(%s, INT8)"); - put(double[].class,"VECTOR(%s, FLOAT64)"); - put(Double[].class,"VECTOR(%s, FLOAT64)"); - put(boolean[].class,"VECTOR(%s, BINARY)"); - put(Boolean[].class,"VECTOR(%s, BINARY)"); - */ ->>>>>>> main } }; @@ -114,35 +80,12 @@ public class OracleVectorStoreFieldHelper { put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); put(List.class, OracleDataTypesMapping.JSON); } -<<<<<<< add-oracle-store }; /** * Suffix added to the effective column name to generate the index name for a vector column. */ public static final String VECTOR_INDEX_SUFFIX = "_VECTOR_INDEX"; -======= - - }; - - /** - * Maps vector type to OracleTypes. Only needed if types other than FLOAT_32 are supported. - */ - private static final Map, Integer> mapOracleTypeToVector = new HashMap() { - { - put(float[].class, OracleTypes.VECTOR_FLOAT32); - put(Float[].class, OracleTypes.VECTOR_FLOAT32); -/* - put(byte[].class, OracleTypes.VECTOR_INT8); - put(Byte[].class, OracleTypes.VECTOR_INT8); - put(Double[].class, OracleTypes.VECTOR_FLOAT64); - put(double[].class, OracleTypes.VECTOR_FLOAT64); - put(Boolean[].class, OracleTypes.VECTOR_BINARY); - put(boolean[].class, OracleTypes.VECTOR_BINARY); -*/ - } - }; ->>>>>>> main /** * Gets the mapping between the supported Java key types and the Oracle database type. @@ -160,20 +103,11 @@ public static HashMap, String> getSupportedKeyTypes() { */ public static Map, String> getSupportedDataTypes( StringTypeMapping stringTypeMapping, int defaultVarCharLength) { -<<<<<<< add-oracle-store String stringType = stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR) ? String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength) : OracleDataTypesMapping.STRING_CLOB; supportedDataTypes.put(String.class, stringType); LOGGER.finest("Mapping String columns to " + stringType); -======= - - if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { - supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); - } else { - supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); - } ->>>>>>> main return supportedDataTypes; } @@ -224,22 +158,14 @@ public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField */ public static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { -<<<<<<< add-oracle-store String dataFieldIndex = "CREATE MULTIVALUE INDEX IF NOT EXISTS %s ON %s t (t.%s.%s)"; -======= - String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; ->>>>>>> main return String.format(dataFieldIndex, collectionTableName + "_" + dataField.getEffectiveStorageName(), collectionTableName, dataField.getEffectiveStorageName(), getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); } else { -<<<<<<< add-oracle-store String dataFieldIndex = "CREATE INDEX IF NOT EXISTS %s ON %s (%s ASC)"; -======= - String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; ->>>>>>> main return String.format(dataFieldIndex, collectionTableName + "_" + dataField.getEffectiveStorageName(), collectionTableName, @@ -249,7 +175,6 @@ public static String createIndexForDataField(String collectionTableName, VectorS } /** -<<<<<<< add-oracle-store * Returns vector columns names and types for CREATE TABLE statement * @param fields list of vector record fields. * @return comma separated list of columns and types for CREATE TABLE statement. @@ -294,8 +219,6 @@ private static String getTypeForVectorField(VectorStoreRecordVectorField field) } /** -======= ->>>>>>> main * Gets the function that allows to return the function that converts the JSON value to the * data type. * @param jdbcType The JDBC type. @@ -321,93 +244,4 @@ private static String getFunctionForType(String jdbcType) { } } -<<<<<<< add-oracle-store -======= - /** - * Gets the type of the vector given the field definition. This method is not needed if only - * - * @param field the vector field definition. - * @return returns the type of vector for the given field type. - */ - public static String getTypeForVectorField(VectorStoreRecordVectorField field) { - String dimension = field.getDimensions() > 0 ? String.valueOf(field.getDimensions()) : "*"; - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); -/* Not needed since all types are FLOAT32 - if (field.getFieldSubType() != null) { - String vectorType; - switch (field.getFieldSubType().getName()) { - case "java.lang.Double": - vectorType = "FLOAT64"; - break; - case "java.lang.Byte": - vectorType = "INT8"; - break; - case "java.lang.Boolean": - vectorType = "BINARY"; - break; - default: - vectorType = "FLOAT32"; - } - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension, vectorType); - } else { - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); - } - */ - } - - /** - * Gets the JDBC oracle of the vector field definition. - * @param field the vector field definition. - * @return the JDBC oracle type. - */ - public static int getOracleTypeForField(VectorStoreRecordVectorField field) { - if (field.getFieldSubType() == null) { - return mapOracleTypeToVector.get(field.getFieldType()).intValue(); - } else { - switch (field.getFieldSubType().getName()) { - case "java.lang.Double": - return OracleTypes.VECTOR_FLOAT64; - case "java.lang.Byte": - return OracleTypes.VECTOR_INT8; - case "java.lang.Boolean": - return OracleTypes.VECTOR_BINARY; - default: - return OracleTypes.VECTOR_FLOAT32; - } - } - } - - /** - * Generates the index name given the field name. by suffixing "_VECTOR_INDEX" to the field name. - * @param effectiveStorageName the field name. - * @return the index name. - */ - private static String getIndexName(String effectiveStorageName) { - return effectiveStorageName + "_VECTOR_INDEX"; - } - - /** - * Returns vector columns names and types for CREATE TABLE statement - * @param fields list of vector record fields. - * @return comma separated list of columns and types for CREATE TABLE statement. - */ - public static String getVectorColumnNamesAndTypes(List fields) { - List columns = fields.stream() - .map(field -> field.getEffectiveStorageName() + " " + - OracleVectorStoreFieldHelper.getTypeForVectorField(field) - ).collect(Collectors.toList()); - - return String.join(", ", columns); - } - - /** - * Returns key column names and type for key column for CREATE TABLE statement - * @param field the key field. - * @return column name and type of the key field for CREATE TABLE statement. - */ - public static String getKeyColumnNameAndType(VectorStoreRecordKeyField field) { - return field.getEffectiveStorageName() + " " + supportedKeyTypes.get(field.getFieldType()); - } - ->>>>>>> main } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 9386a744..fd4f9dbf 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -182,7 +182,6 @@ public OracleVectorStoreRecordMapper build() { value = resultSet.getBoolean(field.getEffectiveStorageName()); break; case OracleDataTypesMapping.OFFSET_DATE_TIME: -<<<<<<< add-oracle-store TIMESTAMPTZ timestamptz = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()); value = timestamptz != null ? timestamptz.offsetDateTimeValue() : null; break; @@ -193,16 +192,6 @@ public OracleVectorStoreRecordMapper build() { String uuidValue = resultSet.getString(field.getEffectiveStorageName()); value = uuidValue == null ? null : UUID.fromString(uuidValue); break; -======= - value = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()) - .offsetDateTimeValue(); - break; - case OracleDataTypesMapping.BYTE_ARRAY: - value = resultSet.getBytes(field.getEffectiveStorageName()); - break; - // fallthrough - case OracleDataTypesMapping.UUID: ->>>>>>> main case OracleDataTypesMapping.JSON: value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); break; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java index e0759c4f..f81176b3 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -1,12 +1,9 @@ -<<<<<<< add-oracle-store /* ** Semantic Kernel Oracle connector version 1.0. ** ** Copyright (c) 2025 Oracle and/or its affiliates. ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ */ -======= ->>>>>>> main package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; @@ -128,14 +125,6 @@ void testDataTypesSearch (ClassWithAllBoxedTypes record) { assertEquals(1, results.getTotalCount()); assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); -<<<<<<< add-oracle-store -======= - System.out.println(record.getDecimalValue()); - System.out.println(record.getDecimalValue().doubleValue()); - System.out.println(results.getResults().get(0).getRecord().getDecimalValue()); - System.out.println(results.getResults().get(0).getRecord().getDecimalValue().doubleValue()); - ->>>>>>> main // decimal results = collection.searchAsync( null, diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 88df2cf9..1d66a8e3 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -8,11 +8,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -<<<<<<< add-oracle-store -======= -import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; ->>>>>>> main import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; @@ -23,19 +18,10 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; -<<<<<<< add-oracle-store -======= -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; ->>>>>>> main import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; -<<<<<<< add-oracle-store -======= -import oracle.jdbc.OracleConnection; -import oracle.jdbc.datasource.impl.OracleDataSource; ->>>>>>> main import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -43,31 +29,13 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -<<<<<<< add-oracle-store import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.Statement; -======= -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.time.Duration; -import java.time.OffsetDateTime; ->>>>>>> main import java.util.Arrays; import java.util.HashMap; import java.util.List; -<<<<<<< add-oracle-store -======= -import java.util.Map; -import java.util.UUID; -import java.util.logging.Logger; ->>>>>>> main import java.util.stream.Collectors; import java.util.stream.Stream; @@ -80,63 +48,6 @@ public class OracleVectorStoreRecordCollectionTest extends OracleCommonVectorStoreRecordCollectionTest { private static VectorStoreRecordCollection recordCollection; -<<<<<<< add-oracle-store -======= - private static final String ORACLE_IMAGE_NAME = "gvenzl/oracle-free:23.7-slim-faststart"; - private static final OracleDataSource DATA_SOURCE; - private static final OracleDataSource SYSDBA_DATA_SOURCE; - - static { - - try { - DATA_SOURCE = new oracle.jdbc.datasource.impl.OracleDataSource(); - SYSDBA_DATA_SOURCE = new oracle.jdbc.datasource.impl.OracleDataSource(); - String urlFromEnv = System.getenv("ORACLE_JDBC_URL"); - - if (urlFromEnv == null) { - // The Ryuk component is relied upon to stop this container. - OracleContainer oracleContainer = new OracleContainer(ORACLE_IMAGE_NAME) - .withCopyFileToContainer(MountableFile.forClasspathResource("/initialize.sql"), - "/container-entrypoint-initdb.d/initialize.sql") - .withStartupTimeout(Duration.ofSeconds(600)) - .withConnectTimeoutSeconds(600) - .withDatabaseName("pdb1") - .withUsername("testuser") - .withPassword("testpwd"); - oracleContainer.start(); - - initDataSource( - DATA_SOURCE, - oracleContainer.getJdbcUrl(), - oracleContainer.getUsername(), - oracleContainer.getPassword()); - initDataSource(SYSDBA_DATA_SOURCE, oracleContainer.getJdbcUrl(), "sys", oracleContainer.getPassword()); - } else { - initDataSource( - DATA_SOURCE, - urlFromEnv, - System.getenv("ORACLE_JDBC_USER"), - System.getenv("ORACLE_JDBC_PASSWORD")); - initDataSource( - SYSDBA_DATA_SOURCE, - urlFromEnv, - System.getenv("ORACLE_JDBC_USER"), - System.getenv("ORACLE_JDBC_PASSWORD")); - } - SYSDBA_DATA_SOURCE.setConnectionProperty(OracleConnection.CONNECTION_PROPERTY_INTERNAL_LOGON, "SYSDBA"); - - } catch (SQLException sqlException) { - throw new AssertionError(sqlException); - } - } - - static void initDataSource(OracleDataSource dataSource, String url, String username, String password) { - dataSource.setURL(url); - dataSource.setUser(username); - dataSource.setPassword(password); - } - ->>>>>>> main @BeforeAll public static void setup() throws Exception { @@ -187,7 +98,6 @@ private static List getHotels() { Float[] arrayF5 = new Float[] { -3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f }; return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", -<<<<<<< add-oracle-store vec1, arrayf1, arrayf1, arrayF1, 4.0), new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", @@ -201,36 +111,6 @@ private static List getHotels() { 4.0), new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", vec5, arrayf5, arrayf5, arrayF5, -======= - Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), - new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, - new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, - new Float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, - 4.0), - new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", - Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), - new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, - new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, - new Float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, - 4.0), - new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", - Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), - new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, - new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, - new Float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, - 5.0), - new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", - Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), - new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, - new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, - new Float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, - 4.0), - new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", - Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), - new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, - new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, - new Float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, ->>>>>>> main 4.0)); } @@ -477,89 +357,6 @@ void testKeyTypes(String suffix, Class keyType, Object keyValue) { collection.deleteCollectionAsync().block(); } -<<<<<<< add-oracle-store -======= - @ParameterizedTest - @MethodSource("supportedDataTypes") - void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFieldValue, Class fieldSubType) { - VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() - .withName("id") - .withStorageName("id") - .withFieldType(String.class) - .build(); - - VectorStoreRecordDataField dataField; - if (fieldSubType != null) { - dataField = VectorStoreRecordDataField.builder() - .withName("dummy") - .withStorageName("dummy") - .withFieldType(dataFieldType, fieldSubType) - .isFilterable(true) - .build(); - } else { - dataField = VectorStoreRecordDataField.builder() - .withName("dummy") - .withStorageName("dummy") - .withFieldType(dataFieldType) - .isFilterable(true) - .build(); - } - - VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() - .withName("vec") - .withStorageName("vec") - .withFieldType(List.class) - .withDimensions(2) - .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) - .withIndexKind(IndexKind.UNDEFINED) - .build(); - - VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( - Arrays.asList(keyField, dataField, dummyVector) - ); - - OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() - .withDataSource(DATA_SOURCE) - .build(); - - JDBCVectorStore vectorStore = JDBCVectorStore.builder() - .withDataSource(DATA_SOURCE) - .withOptions(JDBCVectorStoreOptions.builder() - .withQueryProvider(queryProvider) - .build()) - .build(); - - String collectionName = "test_datatype_" + dataFieldName; - - VectorStoreRecordCollection collection = - vectorStore.getCollection(collectionName, - JDBCVectorStoreRecordCollectionOptions. builder() - .withRecordClass(DummyRecordForDataTypes.class) - .withRecordDefinition(definition).build()); - - collection.createCollectionAsync().block(); - - String key = "testid"; - - DummyRecordForDataTypes record = - new DummyRecordForDataTypes(key, dataFieldValue, Arrays.asList(1.0f, 2.0f)); - - collection.upsertAsync(record, null).block(); - - DummyRecordForDataTypes result = collection.getAsync(key, null).block(); - assertNotNull(result); - - if (dataFieldValue instanceof Number && result.getDummy() instanceof Number) { - assertEquals(((Number) dataFieldValue).doubleValue(), ((Number) result.getDummy()).doubleValue()); - } else if (dataFieldValue instanceof byte[]) { - assertArrayEquals((byte[]) dataFieldValue, (byte[]) result.getDummy()); - } else { - assertEquals(dataFieldValue, result.getDummy()); - } - - collection.deleteCollectionAsync().block(); - } ->>>>>>> main @Nested class HNSWIndexTests { @@ -738,11 +535,7 @@ private static Stream parametersExactSearch() { // thus upsertAync/getAsync won't work private static Stream supportedKeyTypes() { return Stream.of( -<<<<<<< add-oracle-store Arguments.of("string", String.class, "asd123") /*, -======= - Arguments.of("string", String.class, "asd123")/*, ->>>>>>> main Arguments.of("integer", Integer.class, 321), Arguments.of("long", Long.class, 5L), Arguments.of("short", Short.class, (short) 3), @@ -750,28 +543,6 @@ private static Stream supportedKeyTypes() { ); } -<<<<<<< add-oracle-store -======= - private static Stream supportedDataTypes() { - return Stream.of( - Arguments.of("string", String.class, "asd123", null), - Arguments.of("boolean_true", Boolean.class, true, null), - Arguments.of("boolean_false", Boolean.class, false, null), - Arguments.of("byte", Byte.class, (byte) 127, null), - Arguments.of("short", Short.class, (short) 3, null), - Arguments.of("integer", Integer.class, 321, null), - Arguments.of("long", Long.class, 5L, null), - Arguments.of("float", Float.class, 3.14f, null), - Arguments.of("double", double.class, 3.14159265358d, null), - Arguments.of("decimal", BigDecimal.class, new BigDecimal("12345.67"), null), - //Arguments.of("timestamp", OffsetDateTime.class, OffsetDateTime.now(), null) - //Arguments.of("uuid", UUID.class, UUID.randomUUID(), null) - Arguments.of("byte_array", byte[].class, "abc".getBytes(StandardCharsets.UTF_8), null), - Arguments.of("json", List.class, Arrays.asList("a", "s", "d"), String.class) - ); - } - ->>>>>>> main private static class DummyRecordForKeyTypes { private final Object id; private final String dummy; From 219a4d8f439b7318a35d75f48b31ef0ac567862f Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Thu, 26 Jun 2025 22:42:24 -0700 Subject: [PATCH 37/64] add extended test --- .../oracle/OracleVectorStoreExtendedTest.java | 419 ++++++++++++++++++ 1 file changed, 419 insertions(+) create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java new file mode 100644 index 00000000..7fe66229 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java @@ -0,0 +1,419 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import org.junit.jupiter.api.Test; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OracleVectorStoreExtendedTest extends OracleCommonVectorStoreRecordCollectionTest { + + // Test vector types + @Test + void testUseStringVec() { + VectorStoreRecordCollection collection = + createCollection( + "use_string_vec", + DummyRecordForVecString.class, + null); + + DummyRecordForVecString d1 = new DummyRecordForVecString("id1", "description1", "[1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8]"); + DummyRecordForVecString d2 = new DummyRecordForVecString("id2", "description2", "[1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8]"); + + collection.upsertBatchAsync(Arrays.asList(d1,d2), null).block(); + + DummyRecordForVecString rec = collection.getAsync("id1", + GetRecordOptions.builder().includeVectors(true).build()).block(); + + assertNotNull(rec); + assertEquals("[1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8]", rec.getVec()); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testUseCollectionVec() { + VectorStoreRecordCollection collection = + createCollection( + "use_collection_vec", + DummyRecordForVecCollection.class, + null); + + List v1 = Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10f, -1.3f, 5.5f); + List v2 = Arrays.asList(-2f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f); + DummyRecordForVecCollection d1 = new DummyRecordForVecCollection("id1", "", v1); + DummyRecordForVecCollection d2 = new DummyRecordForVecCollection("id2", "", v2); + + collection.upsertBatchAsync(Arrays.asList(d1,d2), null).block(); + + DummyRecordForVecCollection rec = collection.getAsync("id1", + GetRecordOptions.builder().includeVectors(true).build()).block(); + + assertNotNull(rec); + assertEquals(8, rec.getVec().size()); + assertIterableEquals(v1, rec.getVec()); + + collection.deleteCollectionAsync().block(); + } + + // Test corner-case + @Test + void testUseCLOB() { + VectorStoreRecordCollection collection = + createCollection( + "use_clob", + DummyRecordForCLOB.class, + OracleVectorStoreQueryProvider.StringTypeMapping.USE_CLOB); + + DummyRecordForCLOB d1 = new DummyRecordForCLOB("id1", "clob-description", null); + DummyRecordForCLOB d2 = new DummyRecordForCLOB("id2", "clob-description2", vec(0)); + + collection.upsertBatchAsync(Arrays.asList(d1,d2), null).block(); + + try (Connection c = DATA_SOURCE.getConnection()) { + PreparedStatement st = c.prepareStatement( + "SELECT DATA_TYPE FROM USER_TAB_COLUMNS " + + "WHERE TABLE_NAME = 'SKCOLLECTION_USE_CLOB' AND COLUMN_NAME = 'DESCRIPTION'" + ); + ResultSet rs = st.executeQuery(); + rs.next(); + assertEquals("CLOB", rs.getString(1)); + } catch (SQLException e) { + throw new RuntimeException(e); + } finally { + collection.deleteCollectionAsync().block(); + } + } + + @Test + void testClobLongText() { + VectorStoreRecordCollection collection = + createCollection( + "clob_long_text", + DummyRecordForCLOB.class, + OracleVectorStoreQueryProvider.StringTypeMapping.USE_CLOB); + + String longText = String.join("", java.util.Collections.nCopies(6000, "a")); + DummyRecordForCLOB r = new DummyRecordForCLOB("big", longText, vec(0)); + collection.upsertAsync(r, null).block(); + + DummyRecordForCLOB out = collection.getAsync("big", null).block(); + assertEquals(longText.length(), out.getDescription().length()); + assertTrue(out.getDescription().startsWith("aaaa")); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testMultipleFilter() { + VectorStoreRecordCollection collection = + createCollection( + "multiple_filter", + DummyRecordForMultipleFilter.class, + null); + + DummyRecordForMultipleFilter d1 = new DummyRecordForMultipleFilter("id1", 4, 120, floatVec(0f)); + DummyRecordForMultipleFilter d2 = new DummyRecordForMultipleFilter("id2", 4, 100, floatVec(0f)); + DummyRecordForMultipleFilter d3 = new DummyRecordForMultipleFilter("id3", 3, 100, floatVec(0f)); + + collection.upsertBatchAsync(Arrays.asList(d1,d2,d3), null).block(); + + VectorSearchFilter filter = VectorSearchFilter.builder() + .equalTo("price",100) + .equalTo("stars", 4) + .build(); + + VectorSearchResults results = + collection.searchAsync(null, + VectorSearchOptions.builder() + .withVectorSearchFilter(filter) + .build() + ).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals("id2", results.getResults().get(0).getRecord().getId()); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testVectorDimensionMismatch() { + VectorStoreRecordCollection collection = + createCollection( + "vector_dimension_mismatch", + DummyRecord.class, + null); + + // Empty vector rejected + DummyRecord d1 = new DummyRecord("id1", 4, 120d, new float[]{}); + SKException ex = assertThrows(SKException.class, + () -> collection.upsertBatchAsync(Arrays.asList(d1), null).block()); + System.out.println(ex.getMessage()); + assertTrue(ex.getCause().getMessage().contains("ORA-51803")); + + // Vector dimension mismatch + DummyRecord d2 = new DummyRecord("id1", 4, 120d, new float[]{1.1f,2.2f,3.3f,4.4f,5.5f}); + SKException ex2 = assertThrows(SKException.class, + () -> collection.upsertBatchAsync(Arrays.asList(d2), null).block()); + assertTrue(ex2.getCause().getMessage().contains("ORA-51803")); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testNull() { + VectorStoreRecordCollection collection = + createCollection("test_null", DummyRecord.class, null); + + DummyRecord d1 = new DummyRecord("id1", 4, null, floatVec(1)); + collection.upsertBatchAsync(Arrays.asList(d1), null).block(); + + VectorSearchFilter filter = VectorSearchFilter.builder() + .equalTo("price",null)// + .build(); + + VectorSearchResults results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter(filter) + .build() + ).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals("id1", results.getResults().get(0).getRecord().getId()); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testSkipAndTop() { + + } + + private VectorStoreRecordCollection createCollection( + String collectionName, + Class recordClass, + OracleVectorStoreQueryProvider.StringTypeMapping stringTypeMapping) { + + OracleVectorStoreQueryProvider.Builder builder = + OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE); + + if (stringTypeMapping != null) { + builder.withStringTypeMapping(stringTypeMapping); + } + OracleVectorStoreQueryProvider queryProvider = builder.build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(recordClass) + .build()).createCollectionAsync().block(); + + return collection; + } + + private List vec(float x) { + return Arrays.asList(x, x+1, x+2, x+3, x+4, x+5, x+6, x+7); + } + + private float[] floatVec(float x) { + return new float[] { x, x+1, x+2, x+3, x+4, x+5, x+6, x+7 }; + } + + private static class DummyRecordForVecString { + @VectorStoreRecordKey + private final String id; + + @VectorStoreRecordData(isFilterable = false) + private final String description; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private final String vec; + + public DummyRecordForVecString() { + this(null, null, null); + } + public DummyRecordForVecString(String id, String description, String vec) { + this.id = id; + this.description = description; + this.vec = vec; + } + + public String getId() { + return id; + } + public String getDescription() { + return description; + } + public String getVec() { + return vec; + } + } + + private static class DummyRecordForVecCollection{ + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = false) + private String description; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private Collection vec; + + public DummyRecordForVecCollection() { + this(null, null, null); + } + public DummyRecordForVecCollection(String id, String description, Collection vec) { + this.id = id; + this.description = description; + this.vec = vec; + } + + public String getId() { + return id; + } + public String getDescription() { + return description; + } + public Collection getVec() { + return vec; + } + } + + private static class DummyRecordForCLOB { + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = false) + private String description; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private List vec; + + private DummyRecordForCLOB() { + this(null, null, null); + } + private DummyRecordForCLOB(String id, String description, List vec) { + this.id = id; + this.description = description; + this.vec = vec; + } + + public String getId() { + return id; + } + public String getDescription() { + return description; + } + public List getVec() { + return vec; + } + } + + private static class DummyRecordForMultipleFilter { + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = true) + private int stars; + + @VectorStoreRecordData(isFilterable = true) + private double price; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private float[] vec; + + public DummyRecordForMultipleFilter() { + this(null, 0, 0d, null); + } + + public DummyRecordForMultipleFilter(String id, int stars, double price, float[] vec) { + this.id = id; + this.stars = stars; + this.price = price; + this.vec = vec; + } + + public String getId() { + return id; + } + public int getStars() { + return stars; + } + public double getPrice() { + return price; + } + public float[] getVec() { + return vec; + } + } + + private static class DummyRecord { + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = true) + private int stars; + + @VectorStoreRecordData(isFilterable = true) + private Double price; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private float[] vec; + + public DummyRecord() { + this(null, 0, 0d, null); + } + + public DummyRecord(String id, int stars, Double price, float[] vec) { + this.id = id; + this.stars = stars; + this.price = price; + this.vec = vec; + } + + public String getId() { + return id; + } + public int getStars() { + return stars; + } + public Double getPrice() { + return price; + } + public float[] getVec() { + return vec; + } + } +} From 756e1adc6b7b0a6e2e6b5cba1e323d0428899bbf Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Thu, 26 Jun 2025 23:08:11 -0700 Subject: [PATCH 38/64] resolve conflict of git rebase --- .../OracleVectorStoreQueryProvider.java | 21 +++++++++++++++++++ .../oracle/OracleVectorStoreRecordMapper.java | 14 +++++++++++++ 2 files changed, 35 insertions(+) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 9c347a5e..151a1c1e 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -45,6 +45,7 @@ import java.sql.SQLException; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -328,6 +329,25 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object // Some field types require special treatment to convert the java type to the // DB type if (field instanceof VectorStoreRecordVectorField) { + + // Convert the vector field to a string + if (field.getFieldType().equals(String.class)) { + String json = (valueNode == null || valueNode.isNull()) + ? null + : valueNode.asText(); + double[] values = (json == null) + ? null + : objectMapper.readValue(json, double[].class); + + int dim = ((VectorStoreRecordVectorField) field).getDimensions(); + if (values != null && values.length != dim) { + throw new SKException("Vector dimension mismatch: expected " + dim); + } + + upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); + continue; + } + // If the vector field is not set as a string convert to an array of doubles // and set the value if (!field.getFieldType().equals(String.class)) { @@ -336,6 +356,7 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object : StreamSupport.stream(( (ArrayNode)valueNode).spliterator(), false) .mapToDouble(d -> d.asDouble()).toArray(); + upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); continue; } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index fd4f9dbf..2c3b3db3 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -6,6 +6,7 @@ */ package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -210,6 +211,17 @@ public OracleVectorStoreRecordMapper build() { } if (options != null && options.isIncludeVectors()) { for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + + // String vector + if (field.getFieldType().equals(String.class)) { + float[] arr = resultSet.getObject(field.getEffectiveStorageName(), float[].class); + String str = (arr == null) + ? null + : objectMapper.writeValueAsString(arr); + objectNode.put(field.getEffectiveStorageName(), str); + continue; + } + Object value = resultSet.getObject(field.getEffectiveStorageName(), float[].class); JsonNode genericNode = objectMapper.valueToTree(value); objectNode.set(field.getEffectiveStorageName(), genericNode); @@ -227,6 +239,8 @@ public OracleVectorStoreRecordMapper build() { throw new SKException( "Failure to serialize object, by default the JDBC connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", e); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); } }); } From 578c733d48b4f17a96b695d8d76c4c81aef344c0 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Fri, 27 Jun 2025 10:34:32 +0200 Subject: [PATCH 39/64] Changes after Michael's code review --- .../OracleVectorStoreQueryProvider.java | 68 +++++++++---------- .../oracle/OracleVectorStoreRecordMapper.java | 5 +- ...OracleVectorStoreRecordCollectionTest.java | 2 +- 3 files changed, 33 insertions(+), 42 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 151a1c1e..30c52ce3 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -6,7 +6,9 @@ */ package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -33,10 +35,13 @@ import oracle.jdbc.OracleStatement; import oracle.jdbc.OracleTypes; import oracle.sql.TIMESTAMPTZ; +import oracle.jdbc.provider.oson.OsonFactory; import javax.annotation.Nonnull; import javax.annotation.concurrent.GuardedBy; import javax.sql.DataSource; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; @@ -48,12 +53,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.StreamSupport; /** * JDBC Vector Store for the Oracle Database @@ -329,42 +332,33 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object // Some field types require special treatment to convert the java type to the // DB type if (field instanceof VectorStoreRecordVectorField) { - - // Convert the vector field to a string - if (field.getFieldType().equals(String.class)) { - String json = (valueNode == null || valueNode.isNull()) - ? null - : valueNode.asText(); - double[] values = (json == null) - ? null - : objectMapper.readValue(json, double[].class); - - int dim = ((VectorStoreRecordVectorField) field).getDimensions(); - if (values != null && values.length != dim) { - throw new SKException("Vector dimension mismatch: expected " + dim); - } - - upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); - continue; - } - - // If the vector field is not set as a string convert to an array of doubles + // If the vector field is not set as a string convert to an array of floats // and set the value if (!field.getFieldType().equals(String.class)) { - double[] values = (valueNode == null || valueNode.isNull()) - ? null - : StreamSupport.stream(( - (ArrayNode)valueNode).spliterator(), false) - .mapToDouble(d -> d.asDouble()).toArray(); - - upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); + if (valueNode != null && !valueNode.isNull() && valueNode.isArray()) { + final float[] values = new float[valueNode.size()]; + for (int j = 0; j < ((ArrayNode)valueNode).size(); j++) { + values[j] = ((ArrayNode)valueNode).get(j).floatValue(); + } + upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); + } else { + upsertStatement.setNull(i + 1, OracleTypes.VECTOR_FLOAT32); + } continue; } } else if (field instanceof VectorStoreRecordDataField) { - // Lists are stored as JSON objects, write the list as a JSON string representation - // of the list. + // Lists are stored as JSON objects, write the list using the JDBC OSON + // extensions. if (field.getFieldType().equals(List.class)) { - upsertStatement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + JsonFactory osonFactory = new OsonFactory(); + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + try (JsonGenerator osonGen = osonFactory.createGenerator(out)) { + objectMapper.writeValue(osonGen, valueNode); + } + upsertStatement.setBytes(i + 1, out.toByteArray()); + } catch (IOException ioEx) { + throw new SKException("Failed to convert list to JSON value", ioEx); + } continue; } // Convert UUID to string before setting the value. @@ -372,14 +366,14 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object upsertStatement.setObject(i + 1, valueNode.isNull() ? null : valueNode.asText()); continue; } - // Convert OffsetDateTime to TIMESTAMPTZ before setting the value. + // Convert value node (its representations depends on Jackson JSON features) + // to OffsetDateTime before setting the value. if (field.getFieldType().equals(OffsetDateTime.class)) { if (valueNode == null || valueNode.isNull()) { upsertStatement.setNull(i + 1, OracleTypes.TIMESTAMPTZ); } else { OffsetDateTime offsetDateTime = (OffsetDateTime) objectMapper.convertValue(valueNode, field.getFieldType()); - ((OraclePreparedStatement) upsertStatement).setTIMESTAMPTZ(i + 1, - TIMESTAMPTZ.of(offsetDateTime)); + upsertStatement.setObject(i + 1, offsetDateTime); } continue; } @@ -388,8 +382,8 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object // For all other field type use setObject with the field value upsertStatement.setObject(i + 1, objectMapper.convertValue(valueNode,field.getFieldType())); - } catch (SQLException | JsonProcessingException e) { - throw new RuntimeException(e); + } catch (SQLException e) { + throw new SKException(e); } } } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 2c3b3db3..2575261c 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -18,9 +18,7 @@ import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import oracle.jdbc.OracleResultSet; import oracle.jdbc.provider.oson.OsonModule; -import oracle.sql.TIMESTAMPTZ; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Map; @@ -183,8 +181,7 @@ public OracleVectorStoreRecordMapper build() { value = resultSet.getBoolean(field.getEffectiveStorageName()); break; case OracleDataTypesMapping.OFFSET_DATE_TIME: - TIMESTAMPTZ timestamptz = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()); - value = timestamptz != null ? timestamptz.offsetDateTimeValue() : null; + value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); break; case OracleDataTypesMapping.BYTE_ARRAY: value = resultSet.getBytes(field.getEffectiveStorageName()); diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 1d66a8e3..a2bd7af8 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -290,7 +290,7 @@ public void searchWithTagFilter() { .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); assertNotNull(results); assertEquals(1, results.size()); - // The first hotel should be the most similar + // The second hotel contains the tag we are searching for assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); } From dd6be5e6adfa90e9e0e6791a5b4d1f5b4698e211 Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 28 Jun 2025 00:09:21 -0700 Subject: [PATCH 40/64] add more test to extended test --- .../oracle/OracleVectorStoreExtendedTest.java | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java index 7fe66229..834c52c1 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; @@ -11,17 +12,23 @@ import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; + import org.junit.jupiter.api.Test; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertIterableEquals; @@ -184,7 +191,7 @@ void testVectorDimensionMismatch() { } @Test - void testNull() { + void testNullFieldValue() { VectorStoreRecordCollection collection = createCollection("test_null", DummyRecord.class, null); @@ -210,7 +217,59 @@ void testNull() { @Test void testSkipAndTop() { + VectorStoreRecordCollection collection = + createCollection( + "test_skip_and_top", + DummyRecord.class, + null); + + List l1 = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + l1.add(new DummyRecord("id" + i, i, (double) i, floatVec(i))); + } + collection.upsertBatchAsync(l1, null).block(); + + VectorSearchResults results = collection.searchAsync( + Collections.nCopies(8,0f), + VectorSearchOptions.builder() + .withIncludeVectors(true) + .withSkip(5) + .withTop(3) + .build() + ).block(); + + assertEquals(3, results.getResults().size()); + List ids = results.getResults().stream().map(r -> r.getRecord().getId()).collect( + Collectors.toList()); + assertEquals(Arrays.asList("id6","id7","id8"), ids); + + collection.deleteCollectionAsync().block(); + } + + // corner case for OracleVectorStoreRecordMapper + @Test + void testMapRecordToStorageModel_throws() { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDefinition definition = + VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField) + ); + + OracleVectorStoreRecordMapper mapper = + OracleVectorStoreRecordMapper. builder() + .withRecordClass(DummyRecord.class) + .withVectorStoreRecordDefinition(definition) + .build(); + UnsupportedOperationException ex = assertThrows( + UnsupportedOperationException.class, + () -> mapper.mapRecordToStorageModel(new DummyRecord())); + assertEquals("Not implemented", ex.getMessage()); } private VectorStoreRecordCollection createCollection( From f5c9cfab57a94dbcb754e22bbbaae2260ddec09f Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 1 Jul 2025 21:55:24 +0200 Subject: [PATCH 41/64] Set search vector as array of float[] insteand of List as string --- .../data/jdbc/oracle/OracleVectorStoreQueryProvider.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 30c52ce3..aeda5177 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -450,8 +450,11 @@ public VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List(mapper.mapStorageModelToRecord(rs, getRecordOptions), score)); } } - } catch (SQLException | JsonProcessingException e) { + } catch (SQLException e) { throw new SKException("Search failed", e); } From 2a8d37240271e2a1c50ca00e5bc6dee4bce2f407 Mon Sep 17 00:00:00 2001 From: psilberk Date: Tue, 1 Jul 2025 16:04:28 -0700 Subject: [PATCH 42/64] Clean up and remove deps from pom --- data/semantickernel-data-oracle/pom.xml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index b383e16d..f777f72f 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -43,17 +43,6 @@ jackson-core compile - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - 2.18.0 - - com.oracle.database.jdbc ojdbc11 @@ -64,7 +53,6 @@ ojdbc-provider-jackson-oson 1.0.4 - org.junit.jupiter junit-jupiter From c793b82a2284c4caef00f847bff47e261c7b5385 Mon Sep 17 00:00:00 2001 From: psilberk Date: Tue, 1 Jul 2025 22:29:17 -0700 Subject: [PATCH 43/64] Remove unused imports --- .../oracle/OracleVectorStoreFieldHelper.java | 2 -- .../OracleVectorStoreQueryProvider.java | 23 ++++++++----------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index c177131f..7fe4895b 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -8,10 +8,8 @@ import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; -import oracle.jdbc.OracleTypes; import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.Collection; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index aeda5177..9de32ef3 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -7,7 +7,6 @@ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.MapperFeature; @@ -16,7 +15,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; -import com.microsoft.semantickernel.data.jdbc.*; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; @@ -31,32 +30,30 @@ import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import oracle.jdbc.OraclePreparedStatement; -import oracle.jdbc.OracleStatement; -import oracle.jdbc.OracleTypes; -import oracle.sql.TIMESTAMPTZ; -import oracle.jdbc.provider.oson.OsonFactory; - -import javax.annotation.Nonnull; -import javax.annotation.concurrent.GuardedBy; -import javax.sql.DataSource; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.Statement; import java.sql.SQLException; +import java.sql.Statement; import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.UUID; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Logger; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.concurrent.GuardedBy; +import javax.sql.DataSource; +import oracle.jdbc.OraclePreparedStatement; +import oracle.jdbc.OracleStatement; +import oracle.jdbc.OracleTypes; +import oracle.jdbc.provider.oson.OsonFactory; +import oracle.sql.TIMESTAMPTZ; /** * JDBC Vector Store for the Oracle Database From 3b299ee5c1b1f26272b69fa541c4d948330f09b1 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 7 Jul 2025 18:37:54 +0200 Subject: [PATCH 44/64] Copyright and license --- .../jdbc/oracle/OracleDataTypesMapping.java | 25 ++++++++++++++-- .../oracle/OracleVectorStoreFieldHelper.java | 25 ++++++++++++++-- .../OracleVectorStoreQueryProvider.java | 25 ++++++++++++++-- .../oracle/OracleVectorStoreRecordMapper.java | 25 ++++++++++++++-- .../jdbc/oracle/ClassWithAllBoxedTypes.java | 25 ++++++++++++++-- .../oracle/ClassWithAllPrimitiveTypes.java | 25 ++++++++++++++-- .../jdbc/oracle/ClassWithAnnotatedTypes.java | 25 ++++++++++++++-- .../data/jdbc/oracle/Hotel.java | 29 ++++++++++++++++--- ...CommonVectorStoreRecordCollectionTest.java | 25 ++++++++++++++-- .../OracleVectorStoreAnnotatedTypeTest.java | 25 ++++++++++++++-- .../OracleVectorStoreDataTypeSearchTest.java | 25 ++++++++++++++-- .../oracle/OracleVectorStoreDataTypeTest.java | 25 ++++++++++++++-- .../oracle/OracleVectorStoreExtendedTest.java | 25 ++++++++++++++++ ...OracleVectorStoreRecordCollectionTest.java | 25 ++++++++++++++-- 14 files changed, 314 insertions(+), 40 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index cfca9ecc..efde970c 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index 7fe4895b..369a036a 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 9de32ef3..239be7a0 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 2575261c..c8ca4b32 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java index 307c974d..14b96c14 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java index 0ca52d7b..6a8d76d0 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java index 5964cb3c..75190deb 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index dc736ba3..19d31071 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -1,7 +1,28 @@ -// Copyright (c) Microsoft. All rights reserved. - -// Should we change the copyright from the file we copied? Or should we -// just change the test to use a class that we create? +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java index 7549188b..ce260c77 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java index 42036826..e0332950 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java index f81176b3..bac0ee4c 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java index cc8d2462..eeb6a2e7 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java index 834c52c1..50c0a608 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java @@ -1,3 +1,28 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index a2bd7af8..41bb0cee 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; From 632e38867f3b010b9c34bac1d7b04c75926b4bb3 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 7 Jul 2025 18:43:46 +0200 Subject: [PATCH 45/64] Samples --- .../data/jdbc/oracle/Hotel.java | 26 +------------------ ...OracleVectorStoreRecordCollectionTest.java | 25 ------------------ .../data/vectorstores/oracle/Book.java | 26 ++++++++++++++++++- .../data/vectorstores/oracle/Main.java | 26 ++++++++++++++++++- 4 files changed, 51 insertions(+), 52 deletions(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index 19d31071..0f93ff7f 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -1,28 +1,4 @@ -/* - ** Oracle Database Vector Store Connector for Semantic Kernel (Java) - ** - ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. - ** - ** The MIT License (MIT) - ** - ** Permission is hereby granted, free of charge, to any person obtaining a copy - ** of this software and associated documentation files (the "Software"), to - ** deal in the Software without restriction, including without limitation the - ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - ** sell copies of the Software, and to permit persons to whom the Software is - ** furnished to do so, subject to the following conditions: - ** - ** The above copyright notice and this permission notice shall be included in - ** all copies or substantial portions of the Software. - ** - ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - ** IN THE SOFTWARE. - */ + package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 41bb0cee..a6a197ec 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -1,28 +1,3 @@ -/* - ** Oracle Database Vector Store Connector for Semantic Kernel (Java) - ** - ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. - ** - ** The MIT License (MIT) - ** - ** Permission is hereby granted, free of charge, to any person obtaining a copy - ** of this software and associated documentation files (the "Software"), to - ** deal in the Software without restriction, including without limitation the - ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - ** sell copies of the Software, and to permit persons to whom the Software is - ** furnished to do so, subject to the following conditions: - ** - ** The above copyright notice and this permission notice shall be included in - ** all copies or substantial portions of the Software. - ** - ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - ** IN THE SOFTWARE. - */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java index 87104cf8..8de0b5aa 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java @@ -1,4 +1,28 @@ -// Copyright (c) Microsoft. All rights reserved. +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java index 570dfc60..c03f3eb3 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -1,4 +1,28 @@ -// Copyright (c) Microsoft. All rights reserved. +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; From 68f7914a46ce0d8a4a018f305a636f0e4ed04962 Mon Sep 17 00:00:00 2001 From: psilberk Date: Wed, 9 Jul 2025 14:34:30 -0700 Subject: [PATCH 46/64] Addressing review comments --- pom.xml | 1 - .../semantickernel-syntax-examples/pom.xml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 3fb4158d..b52f7484 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,6 @@ semantickernel-bom semantickernel-api semantickernel-experimental - samples aiservices/openai aiservices/google aiservices/huggingface diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index d5471730..2392ee65 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -139,7 +139,7 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc - 1.4.4-SNAPSHOT + ${project.version} com.microsoft.semantic-kernel From e6690176273f34fe23d360d65d2a6f1376940615 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Fri, 25 Jul 2025 17:24:19 +0200 Subject: [PATCH 47/64] Addressed some of the review comments --- .vscode/settings.json | 3 +- data/semantickernel-data-jdbc/pom.xml | 5 +- .../jdbc/JDBCVectorStoreQueryProvider.java | 2 +- .../oracle/OracleVectorStoreFieldHelper.java | 72 ++++--- .../OracleVectorStoreQueryProvider.java | 45 ++++- .../oracle/OracleVectorStoreRecordMapper.java | 183 +++++++++--------- .../semantickernel-syntax-examples/pom.xml | 2 +- 7 files changed, 185 insertions(+), 127 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 16667e41..6f3bd17c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -21,5 +21,6 @@ }, }, "java.debug.settings.onBuildFailureProceed": true, - "java.compile.nullAnalysis.mode": "disabled" + "java.compile.nullAnalysis.mode": "disabled", + "java.configuration.updateBuildConfiguration": "interactive" } diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index e9b18f75..592cbbe5 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC2-SNAPSHOT ../../pom.xml @@ -15,9 +15,8 @@ com.microsoft.semantic-kernel - semantickernel-api + semantickernel-api-data - org.slf4j slf4j-api diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index 9381ef06..f19d0c22 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -106,7 +106,7 @@ public JDBCVectorStoreQueryProvider( @SuppressFBWarnings("EI_EXPOSE_REP2") @Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables, - @Nonnull HashMap, String> supportedKeyTypes, + @Nonnull Map, String> supportedKeyTypes, @Nonnull Map, String> supportedDataTypes, @Nonnull Map, String> supportedVectorTypes) { this.dataSource = dataSource; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index 369a036a..d6592215 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -29,9 +29,11 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.exceptions.SKException; import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -45,6 +47,10 @@ */ class OracleVectorStoreFieldHelper { + /** + * Object naming regular expression + */ + private static final String OBJECT_NAMING_REGEXP = "[a-zA-Z_][a-zA-Z0-9_]{1,128}"; /** * The logger */ @@ -109,8 +115,9 @@ class OracleVectorStoreFieldHelper { * * @return the mapping between the supported Java key types and the Oracle database type. */ - public static HashMap, String> getSupportedKeyTypes() { - return supportedKeyTypes; + static Map, String> getSupportedKeyTypes() { + + return Collections.unmodifiableMap(supportedKeyTypes); } /** @@ -118,14 +125,14 @@ public static HashMap, String> getSupportedKeyTypes() { * * @return the mapping between the supported Java data types and the Oracle database type. */ - public static Map, String> getSupportedDataTypes( + static Map, String> getSupportedDataTypes( StringTypeMapping stringTypeMapping, int defaultVarCharLength) { String stringType = stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR) ? String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength) : OracleDataTypesMapping.STRING_CLOB; supportedDataTypes.put(String.class, stringType); LOGGER.finest("Mapping String columns to " + stringType); - return supportedDataTypes; + return Collections.unmodifiableMap(supportedDataTypes); } /** @@ -133,8 +140,9 @@ public static Map, String> getSupportedDataTypes( * * @return the mapping between the supported Java data types and the Oracle database type. */ - public static Map, String> getSupportedVectorTypes() { - return supportedVectorTypes; + static Map, String> getSupportedVectorTypes() { + + return Collections.unmodifiableMap(supportedVectorTypes); } /** @@ -143,20 +151,23 @@ public static Map, String> getSupportedVectorTypes() { * @return the CREATE VECTOR INDEX statement to create the index according to the vector * field definition. */ - public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField field, String collectionTableName) { + static String getCreateVectorIndexStatement(VectorStoreRecordVectorField field, String collectionTableName) { switch (field.getIndexKind()) { case IVFFLAT: return "CREATE VECTOR INDEX IF NOT EXISTS " - + getIndexName(field.getEffectiveStorageName()) + + validateObjectNaming(getIndexName(field.getEffectiveStorageName())) + " ON " - + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + validateObjectNaming(collectionTableName) + + "( " + validateObjectNaming(field.getEffectiveStorageName()) + " ) " + " ORGANIZATION NEIGHBOR PARTITIONS " + " WITH DISTANCE COSINE " + "PARAMETERS ( TYPE IVF )"; case HNSW: - return "CREATE VECTOR INDEX IF NOT EXISTS " + getIndexName(field.getEffectiveStorageName()) + return "CREATE VECTOR INDEX IF NOT EXISTS " + + validateObjectNaming(getIndexName(field.getEffectiveStorageName())) + " ON " - + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + validateObjectNaming(collectionTableName) + + "( " + validateObjectNaming(field.getEffectiveStorageName()) + " ) " + "ORGANIZATION INMEMORY GRAPH " + "WITH DISTANCE COSINE " + "PARAMETERS (TYPE HNSW)"; @@ -173,20 +184,20 @@ public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField * * @return the CREATE INDEX statement to create the index according to the field definition. */ - public static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { + static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { String dataFieldIndex = "CREATE MULTIVALUE INDEX IF NOT EXISTS %s ON %s t (t.%s.%s)"; return String.format(dataFieldIndex, - collectionTableName + "_" + dataField.getEffectiveStorageName(), - collectionTableName, - dataField.getEffectiveStorageName(), + validateObjectNaming(collectionTableName + "_" + dataField.getEffectiveStorageName()), + validateObjectNaming(collectionTableName), + validateObjectNaming(dataField.getEffectiveStorageName()), getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); } else { String dataFieldIndex = "CREATE INDEX IF NOT EXISTS %s ON %s (%s ASC)"; return String.format(dataFieldIndex, - collectionTableName + "_" + dataField.getEffectiveStorageName(), - collectionTableName, - dataField.getEffectiveStorageName() + validateObjectNaming(collectionTableName + "_" + dataField.getEffectiveStorageName()), + validateObjectNaming(collectionTableName), + validateObjectNaming(dataField.getEffectiveStorageName()) ); } } @@ -196,9 +207,9 @@ public static String createIndexForDataField(String collectionTableName, VectorS * @param fields list of vector record fields. * @return comma separated list of columns and types for CREATE TABLE statement. */ - public static String getVectorColumnNamesAndTypes(List fields) { + static String getVectorColumnNamesAndTypes(List fields) { List columns = fields.stream() - .map(field -> field.getEffectiveStorageName() + " " + + .map(field -> validateObjectNaming(field.getEffectiveStorageName()) + " " + OracleVectorStoreFieldHelper.getTypeForVectorField(field) ).collect(Collectors.toList()); @@ -210,8 +221,8 @@ public static String getVectorColumnNamesAndTypes(List VectorSearchResults search(String collectionName, List mapper) { + + if (vector != null && recordDefinition.getVectorFields().isEmpty()) { + throw new SKException("Record definition must contain at least one vector field" + + " to perform a vector search"); + } + // Gets the search vector field and its distance function. If not vector field was provided, // use the first one - VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null - ? recordDefinition.getVectorFields().get(0) - : (VectorStoreRecordVectorField) recordDefinition - .getField(options.getVectorFieldName()); - DistanceFunction distanceFunction = vectorField == null ? null : vectorField.getDistanceFunction(); - if (options.getVectorFieldName() != null && vectorField == null) { - throw new SKException(""); + VectorStoreRecordVectorField vectorField = null; + if (vector != null) { + vectorField = getVectorFieldByName(recordDefinition, options.getVectorFieldName()); } + + // get list of fields that should be returned by the query List fields = (options.isIncludeVectors()) ? recordDefinition.getAllFields() @@ -449,7 +453,9 @@ public VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List(records); } + private VectorStoreRecordVectorField getVectorFieldByName( + VectorStoreRecordDefinition recordDefinition, + String name) { + VectorStoreRecordField vectorField; + if (name != null) { + vectorField = recordDefinition.getField(name); + if (vectorField == null) { + throw new SKException("Vector field not found in record definition"); + } + if (!(vectorField instanceof VectorStoreRecordVectorField)) { + throw new SKException("Invalid type"); + } + } else { + if (recordDefinition.getVectorFields().isEmpty()) { + throw new SKException("Record definition should contain at least one vector field"); + } + vectorField = recordDefinition.getVectorFields().get(0); + } + return (VectorStoreRecordVectorField)vectorField; + } + /** * Sets the parameter value * @param statement the statement diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index c8ca4b32..bed0b5ed 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -161,104 +161,109 @@ public OracleVectorStoreRecordMapper build() { return new OracleVectorStoreRecordMapper<>( (resultSet, options) -> { - try { - objectMapper.registerModule(new OsonModule()); - // Create an ObjectNode to hold the values - ObjectNode objectNode = objectMapper.createObjectNode(); + return MapResultSetToRecord(resultSet, options); + }); + } - // Read non vector fields - for (VectorStoreRecordField field : vectorStoreRecordDefinition.getNonVectorFields()) { - Class fieldType = field.getFieldType(); + private Record MapResultSetToRecord(ResultSet resultSet, GetRecordOptions options) { + try { + objectMapper.registerModule(new OsonModule()); + // Create an ObjectNode to hold the values + ObjectNode objectNode = objectMapper.createObjectNode(); - Object value; - switch (supportedDataTypesMapping.get(fieldType)) { - case OracleDataTypesMapping.STRING_CLOB: - value = resultSet.getString(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.BYTE: - value = resultSet.getByte(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.SHORT: - value = resultSet.getShort(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.INTEGER: - value = resultSet.getInt(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.LONG: - value = resultSet.getLong(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.FLOAT: - value = resultSet.getFloat(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.DOUBLE: - value = resultSet.getDouble(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.DECIMAL: - value = resultSet.getBigDecimal(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.BOOLEAN: - value = resultSet.getBoolean(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.OFFSET_DATE_TIME: - value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); - break; - case OracleDataTypesMapping.BYTE_ARRAY: - value = resultSet.getBytes(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.UUID: - String uuidValue = resultSet.getString(field.getEffectiveStorageName()); - value = uuidValue == null ? null : UUID.fromString(uuidValue); - break; - case OracleDataTypesMapping.JSON: - value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); - break; - default: - value = resultSet.getString(field.getEffectiveStorageName()); - } - // Result set getter method sometimes returns a default value when NULL, - // set value to null in that case. - if (resultSet.wasNull()) { - value = null; - } + // Read non vector fields + for (VectorStoreRecordField field : vectorStoreRecordDefinition.getNonVectorFields()) { + Class fieldType = field.getFieldType(); - JsonNode genericNode = objectMapper.valueToTree(value); + Object value; + switch (supportedDataTypesMapping.get(fieldType)) { + case OracleDataTypesMapping.STRING_CLOB: + value = resultSet.getString(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.BYTE: + value = resultSet.getByte(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.SHORT: + value = resultSet.getShort(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.INTEGER: + value = resultSet.getInt(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.LONG: + value = resultSet.getLong(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.FLOAT: + value = resultSet.getFloat(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.DOUBLE: + value = resultSet.getDouble(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.DECIMAL: + value = resultSet.getBigDecimal(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.BOOLEAN: + value = resultSet.getBoolean(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.OFFSET_DATE_TIME: + value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); + break; + case OracleDataTypesMapping.BYTE_ARRAY: + value = resultSet.getBytes(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.UUID: + String uuidValue = resultSet.getString(field.getEffectiveStorageName()); + value = uuidValue == null ? null : UUID.fromString(uuidValue); + break; + case OracleDataTypesMapping.JSON: + value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); + break; + default: + value = resultSet.getString(field.getEffectiveStorageName()); + } + // Result set getter method sometimes returns a default value when NULL, + // set value to null in that case. + if (resultSet.wasNull()) { + value = null; + } - objectNode.set(field.getEffectiveStorageName(), genericNode); - } - if (options != null && options.isIncludeVectors()) { - for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + JsonNode genericNode = objectMapper.valueToTree(value); - // String vector - if (field.getFieldType().equals(String.class)) { - float[] arr = resultSet.getObject(field.getEffectiveStorageName(), float[].class); - String str = (arr == null) - ? null - : objectMapper.writeValueAsString(arr); - objectNode.put(field.getEffectiveStorageName(), str); - continue; - } + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + if (options != null && options.isIncludeVectors()) { + for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { - Object value = resultSet.getObject(field.getEffectiveStorageName(), float[].class); - JsonNode genericNode = objectMapper.valueToTree(value); - objectNode.set(field.getEffectiveStorageName(), genericNode); - } - } else { - for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { - JsonNode genericNode = objectMapper.valueToTree(null); - objectNode.set(field.getEffectiveStorageName(), genericNode); - } + // String vector + if (field.getFieldType().equals(String.class)) { + float[] arr = resultSet.getObject(field.getEffectiveStorageName(), float[].class); + String str = (arr == null) + ? null + : objectMapper.writeValueAsString(arr); + objectNode.put(field.getEffectiveStorageName(), str); + continue; } - // Deserialize the object node to the record class - return objectMapper.convertValue(objectNode, recordClass); - } catch (SQLException e) { - throw new SKException( - "Failure to serialize object, by default the JDBC connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", - e); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); + Object value = resultSet.getObject(field.getEffectiveStorageName(), float[].class); + JsonNode genericNode = objectMapper.valueToTree(value); + objectNode.set(field.getEffectiveStorageName(), genericNode); } - }); + } else { + for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + JsonNode genericNode = objectMapper.valueToTree(null); + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + } + + // Deserialize the object node to the record class + return objectMapper.convertValue(objectNode, recordClass); + } catch (SQLException e) { + throw new SKException( + "Failure to serialize object, by default the JDBC connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", + e); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } } } + } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 2392ee65..cb735f9c 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -144,7 +144,7 @@ com.microsoft.semantic-kernel semantickernel-learn-resources - 1.4.4-SNAPSHOT + 1.4.4-RC2-SNAPSHOT compile From 4f919c33502bd5dd3f52ea2307c539e4c592cbbc Mon Sep 17 00:00:00 2001 From: psilberk Date: Wed, 6 Aug 2025 21:15:34 -0700 Subject: [PATCH 48/64] Addressed DoubleBraceInitialization warning/error --- .../oracle/OracleVectorStoreFieldHelper.java | 71 +++++++++---------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index d6592215..bf4dd295 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -59,51 +59,48 @@ class OracleVectorStoreFieldHelper { /** * Maps supported key java classes to Oracle database types */ - private static final HashMap, String> supportedKeyTypes = new HashMap() { - { - put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); - } - }; + private static final HashMap, String> supportedKeyTypes = new HashMap(); + static { + supportedKeyTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); + } /** * Maps supported vector java classes to Oracle database types */ - private static final Map, String> supportedVectorTypes = new HashMap() { - { - put(String.class, OracleDataTypesMapping.VECTOR_FLOAT); - put(List.class, OracleDataTypesMapping.VECTOR_FLOAT); - put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); - put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); - put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); - } - }; + private static final Map, String> supportedVectorTypes = new HashMap(); + static { + supportedVectorTypes.put(String.class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(List.class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); + } /** * Maps supported data java classes to Oracle database types */ - private static final HashMap, String> supportedDataTypes = new HashMap() { - { - put(byte.class, OracleDataTypesMapping.BYTE); - put(Byte.class, OracleDataTypesMapping.BYTE); - put(short.class, OracleDataTypesMapping.SHORT); - put(Short.class, OracleDataTypesMapping.SHORT); - put(int.class, OracleDataTypesMapping.INTEGER); - put(Integer.class, OracleDataTypesMapping.INTEGER); - put(long.class, OracleDataTypesMapping.LONG); - put(Long.class, OracleDataTypesMapping.LONG); - put(Float.class, OracleDataTypesMapping.FLOAT); - put(float.class, OracleDataTypesMapping.FLOAT); - put(Double.class, OracleDataTypesMapping.DOUBLE); - put(double.class, OracleDataTypesMapping.DOUBLE); - put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); - put(Boolean.class, OracleDataTypesMapping.BOOLEAN); - put(boolean.class, OracleDataTypesMapping.BOOLEAN); - put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); - put(UUID.class, OracleDataTypesMapping.UUID); - put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); - put(List.class, OracleDataTypesMapping.JSON); - } - }; + private static final HashMap, String> supportedDataTypes = new HashMap(); + static { + supportedDataTypes.put(byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(Byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(Short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(int.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(Double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); + supportedDataTypes.put(Boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); + supportedDataTypes.put(UUID.class, OracleDataTypesMapping.UUID); + supportedDataTypes.put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); + supportedDataTypes.put(List.class, OracleDataTypesMapping.JSON); + } /** * Suffix added to the effective column name to generate the index name for a vector column. From e55bda371f345023050d7c9376a7684c1f0c33f4 Mon Sep 17 00:00:00 2001 From: psilberk Date: Thu, 7 Aug 2025 16:55:50 -0700 Subject: [PATCH 49/64] Fixed typos in comments --- .../data/jdbc/oracle/OracleVectorStoreQueryProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index b42678a2..e52364c4 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -494,7 +494,7 @@ public VectorSearchResults search(String collectionName, List * @throws SQLException if an error occurs while defining the column type */ private void defineDataColumnType(int columnIndex, OracleStatement statement, Class fieldType) throws SQLException { - // swich between supported classes and define the column type on the statement + // switch between supported classes and define the column type on the statement switch (supportedDataTypes.get(fieldType)) { case OracleDataTypesMapping.STRING_CLOB: statement.defineColumnType(columnIndex, OracleTypes.CLOB, Integer.MAX_VALUE); From 1c556af0eb7c7c621d25d5e2c950832ae4421065 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 11 Aug 2025 18:49:45 +0200 Subject: [PATCH 50/64] Disable Android compatibility check --- data/semantickernel-data-oracle/pom.xml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index f777f72f..6fd20ba1 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -79,4 +79,24 @@ test + + + + org.codehaus.mojo + animal-sniffer-maven-plugin + + + android + test + + check + + + + + true + + + + \ No newline at end of file From 3b49161b1beceb0406ca81e1b6c6e7ba50385550 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 12 Aug 2025 09:47:02 +0200 Subject: [PATCH 51/64] Fixed errors returned by CI on JDK8 --- .../data/jdbc/oracle/OracleVectorStoreQueryProvider.java | 7 +++---- .../data/jdbc/oracle/OracleVectorStoreRecordMapper.java | 9 ++------- .../data/jdbc/oracle/OracleVectorStoreExtendedTest.java | 1 - .../oracle/OracleVectorStoreRecordCollectionTest.java | 3 --- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index e52364c4..e3fa157b 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -245,6 +245,7 @@ public void createCollection(String collectionName, * @param options the options */ @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, @@ -422,6 +423,7 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object * @param the record type */ @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") public VectorSearchResults search(String collectionName, List vector, VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper) { @@ -558,8 +560,6 @@ private void setSearchParameter(PreparedStatement statement, int index, Class // Use JSON string to set lists if (List.class.equals(type)) { statement.setObject(index, objectMapper.writeValueAsString(value)); - System.out.println( - "Set values: " + objectMapper.writeValueAsString(value)); return; } // convert UUID to string @@ -575,7 +575,6 @@ private void setSearchParameter(PreparedStatement statement, int index, Class OffsetDateTime offsetDateTime = (OffsetDateTime) value; ((OraclePreparedStatement) statement).setTIMESTAMPTZ(index, TIMESTAMPTZ.of(offsetDateTime)); - System.out.println("Set values: " + offsetDateTime); } return; } @@ -587,7 +586,6 @@ private void setSearchParameter(PreparedStatement statement, int index, Class BigDecimal bigDecimal = (BigDecimal) value; ((OraclePreparedStatement) statement).setBigDecimal(index, bigDecimal); - System.out.println("Set values: " + bigDecimal); } return; } @@ -825,6 +823,7 @@ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { * @param objectMapper the object mapper * @return the builder */ + @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withObjectMapper( ObjectMapper objectMapper) { this.objectMapper = objectMapper; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index bed0b5ed..3b62f07d 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -41,6 +41,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Map; +import java.util.HashMap; import java.util.UUID; import java.util.function.BiFunction; @@ -91,7 +92,6 @@ public static class Builder private VectorStoreRecordDefinition vectorStoreRecordDefinition; private Map, String> supportedDataTypesMapping; private ObjectMapper objectMapper = new ObjectMapper(); - private Map, String> annotatedTypeMapping; /** * Sets the record class. @@ -137,12 +137,7 @@ public Builder withObjectMapper(ObjectMapper objectMapper) { */ public Builder withSupportedDataTypesMapping( Map, String> supportedDataTypesMapping) { - this.supportedDataTypesMapping = supportedDataTypesMapping; - return this; - } - - public Builder withAnnotatedTypeMapping(Map, String> annotatedTypeMapping) { - this.annotatedTypeMapping = annotatedTypeMapping; + this.supportedDataTypesMapping = new HashMap<>(supportedDataTypesMapping); return this; } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java index 50c0a608..af2e28fd 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java @@ -203,7 +203,6 @@ void testVectorDimensionMismatch() { DummyRecord d1 = new DummyRecord("id1", 4, 120d, new float[]{}); SKException ex = assertThrows(SKException.class, () -> collection.upsertBatchAsync(Arrays.asList(d1), null).block()); - System.out.println(ex.getMessage()); assertTrue(ex.getCause().getMessage().contains("ORA-51803")); // Vector dimension mismatch diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index a6a197ec..238a5ef2 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -215,9 +215,6 @@ public void exactSearch(DistanceFunction distanceFunction, List expected assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar - System.out.println(results.get(0).getScore()); - System.out.println(results.get(1).getScore()); - System.out.println(results.get(2).getScore()); assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); assertEquals(expectedDistance.get(0).doubleValue(), results.get(0).getScore(), 0.0001d); assertEquals(hotels.get(0).getId(), results.get(1).getRecord().getId()); From b3dcd99504b7421872fc999b2998dc6b9978cb03 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 11:17:21 +0200 Subject: [PATCH 52/64] Change timestamp precision --- .../semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index efde970c..1d5db527 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -77,7 +77,7 @@ public class OracleDataTypesMapping { /** * Oracle database type used to map offset date time */ - public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; + public static final String OFFSET_DATE_TIME = "TIMESTAMP(9) WITH TIME ZONE"; /** * Oracle database type used to map UUID */ From d77dee23a426339d2425c72489fa635a9ddfdfd3 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 11:25:17 +0200 Subject: [PATCH 53/64] Changed distance precision in tests --- .../oracle/OracleVectorStoreRecordCollectionTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 238a5ef2..850ae923 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -216,11 +216,11 @@ public void exactSearch(DistanceFunction distanceFunction, List expected assertEquals(3, results.size()); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); - assertEquals(expectedDistance.get(0).doubleValue(), results.get(0).getScore(), 0.0001d); + assertEquals(expectedDistance.get(0).doubleValue(), results.get(0).getScore(), 0.0002d); assertEquals(hotels.get(0).getId(), results.get(1).getRecord().getId()); - assertEquals(expectedDistance.get(1).doubleValue(), results.get(1).getScore(), 0.0001d); + assertEquals(expectedDistance.get(1).doubleValue(), results.get(1).getScore(), 0.0002d); assertEquals(hotels.get(3).getId(), results.get(2).getRecord().getId()); - assertEquals(expectedDistance.get(2).doubleValue(), results.get(2).getScore(), 0.0001d); + assertEquals(expectedDistance.get(2).doubleValue(), results.get(2).getScore(), 0.0002d); options = VectorSearchOptions.builder() .withVectorFieldName(distanceFunction.getValue()) @@ -258,7 +258,7 @@ public void searchWithFilter(DistanceFunction distanceFunction, double expectedD assertEquals(3, results.size()); // The first hotel should be the most similar assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); - assertEquals(results.get(0).getScore(), expectedDistance, 0.0001d); + assertEquals(results.get(0).getScore(), expectedDistance, 0.0002d); } From 912db48aeba662c1c10b7fad46d86b856469d883 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 14:16:33 +0200 Subject: [PATCH 54/64] Lower case on MapResultSetToRecord --- .../data/jdbc/oracle/OracleVectorStoreRecordMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 3b62f07d..c75eb663 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -156,11 +156,11 @@ public OracleVectorStoreRecordMapper build() { return new OracleVectorStoreRecordMapper<>( (resultSet, options) -> { - return MapResultSetToRecord(resultSet, options); + return mapResultSetToRecord(resultSet, options); }); } - private Record MapResultSetToRecord(ResultSet resultSet, GetRecordOptions options) { + private Record mapResultSetToRecord(ResultSet resultSet, GetRecordOptions options) { try { objectMapper.registerModule(new OsonModule()); // Create an ObjectNode to hold the values From cfde37f9e1a3b42e4116eef8286330db3deb00a2 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:01:08 +0200 Subject: [PATCH 55/64] Update samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- .../semantickernel-syntax-examples/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index cb735f9c..cd4a367c 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -150,7 +150,7 @@ com.microsoft.semantic-kernel semantickernel-data-postgres - 1.4.4-RC2-SNAPSHOT + ${project.version} compile From 24490cca39fb6bfba1194f6e7d92cd6164385dc4 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:01:55 +0200 Subject: [PATCH 56/64] Update samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- .../semantickernel-syntax-examples/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index cd4a367c..d4e80015 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -156,7 +156,7 @@ com.microsoft.semantic-kernel semantickernel-data-oracle - 1.4.4-RC2-SNAPSHOT + ${project.version} compile From b854f9b8e4bc5886799c7a02b0cfe17f3932b710 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:02:06 +0200 Subject: [PATCH 57/64] Update samples/semantickernel-learn-resources/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- samples/semantickernel-learn-resources/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 8d3cef89..d0903e4c 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -97,7 +97,7 @@ com.microsoft.semantic-kernel semantickernel-data-postgres - 1.4.4-RC2-SNAPSHOT + ${project.version} compile From ba45bb34c31488663da2de3f5d2e5ba968777e45 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:02:41 +0200 Subject: [PATCH 58/64] Update samples/semantickernel-learn-resources/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- samples/semantickernel-learn-resources/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index d0903e4c..46cc092e 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -49,7 +49,7 @@ com.microsoft.semantic-kernel semantickernel-data-postgres - 1.4.4-RC2-SNAPSHOT + ${project.version} org.apache.logging.log4j From 735587b1a596771f500169fa4cc1bb7e7cd67cad Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:03:02 +0200 Subject: [PATCH 59/64] Update samples/semantickernel-learn-resources/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- samples/semantickernel-learn-resources/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 46cc092e..73cdac1d 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -44,7 +44,7 @@ com.microsoft.semantic-kernel semantickernel-data-oracle - 1.4.4-RC2-SNAPSHOT + ${project.version} com.microsoft.semantic-kernel From f0c9615c5fddab9d308ee396bf5e7d14c9c18f7b Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:03:42 +0200 Subject: [PATCH 60/64] Update samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- .../semantickernel-syntax-examples/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index d4e80015..177a2712 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -144,7 +144,7 @@ com.microsoft.semantic-kernel semantickernel-learn-resources - 1.4.4-RC2-SNAPSHOT + ${project.version} compile From 4d61f13e27114dc4f632d89e03471ea544ec0869 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:53:37 +0200 Subject: [PATCH 61/64] Fix: Run Java Integration Tests and Samples --- api-test/integration-tests/pom.xml | 15 +++++++++++++++ semantickernel-bom/pom.xml | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 433e14a8..94e577f1 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -61,6 +61,21 @@ semantickernel-data-jdbc test + + com.microsoft.semantic-kernel + semantickernel-data-mysql + test + + + com.microsoft.semantic-kernel + semantickernel-data-hsqldb + test + + + com.microsoft.semantic-kernel + semantickernel-data-sqlite + test + com.microsoft.semantic-kernel semantickernel-data-redis diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index ad6d7180..9f56c952 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -106,6 +106,21 @@ semantickernel-data-jdbc ${project.version} + + com.microsoft.semantic-kernel + semantickernel-data-mysql + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-data-sqlite + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-data-hsqldb + ${project.version} + com.microsoft.semantic-kernel semantickernel-data-redis From f78b2ab12cdf07a8bc62f6bf3967cd4ae6f024b2 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 16:47:20 +0200 Subject: [PATCH 62/64] Added oracle store to BOM * integration tests * Added oracle module to BOM --- semantickernel-bom/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 9f56c952..940093a0 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -121,6 +121,11 @@ semantickernel-data-hsqldb ${project.version} + + com.microsoft.semantic-kernel + semantickernel-data-oracle + ${project.version} + com.microsoft.semantic-kernel semantickernel-data-redis From ced785b77309a297b2755dc17bed34ef06521fb7 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 17:07:28 +0200 Subject: [PATCH 63/64] Disable spotless plugin (#15) * integration tests * Added oracle module to BOM * Disable spotless for oracle store and result of spotless apply --- .../jdbc/JDBCVectorStoreQueryProvider.java | 16 ++++++++-------- .../data/jdbc/SQLVectorStoreQueryProvider.java | 4 ++-- data/semantickernel-data-oracle/pom.xml | 7 +++++++ .../PostgreSQLVectorStoreQueryProvider.java | 15 ++++++++------- .../memory/VectorStoreWithOracle.java | 18 +++++++++--------- .../data/vectorstores/oracle/Book.java | 5 +++-- .../VectorStoreRecordDefinition.java | 12 ++++++++++-- 7 files changed, 47 insertions(+), 30 deletions(-) diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index f19d0c22..2fcc2d5b 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -696,18 +696,19 @@ public String getEqualToFilter(EqualToFilterClause filterClause) { @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider - .validateSQLidentifier(filterClause.getFieldName()); + .validateSQLidentifier(filterClause.getFieldName()); return String.format("%s LIKE ?", fieldName); } - + @Override - public VectorStoreRecordMapper getVectorStoreRecordMapper(Class recordClass, - VectorStoreRecordDefinition recordDefinition) { + public VectorStoreRecordMapper getVectorStoreRecordMapper( + Class recordClass, + VectorStoreRecordDefinition recordDefinition) { return JDBCVectorStoreRecordMapper.builder() - .withRecordClass(recordClass) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); } /** @@ -765,5 +766,4 @@ public JDBCVectorStoreQueryProvider build() { } } - } diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java index cadfdaf2..30a535fb 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java @@ -160,8 +160,8 @@ VectorSearchResults search(String collectionName, * @return the record mapper that maps JDBC result sets to the given record. */ VectorStoreRecordMapper getVectorStoreRecordMapper( - final Class recordClass, - final VectorStoreRecordDefinition recordDefinition); + final Class recordClass, + final VectorStoreRecordDefinition recordDefinition); /** * The builder for the JDBC vector store query provider. diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index 6fd20ba1..6514ff0b 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -97,6 +97,13 @@ true + + com.diffplug.spotless + spotless-maven-plugin + + true + + \ No newline at end of file diff --git a/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java index bd8dbba7..1f3273eb 100644 --- a/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -442,18 +442,19 @@ public List getFilterParameters(VectorSearchFilter filter) { @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider - .validateSQLidentifier(filterClause.getFieldName()); + .validateSQLidentifier(filterClause.getFieldName()); return String.format("%s @> ?::jsonb", fieldName); } - + @Override - public VectorStoreRecordMapper getVectorStoreRecordMapper(Class recordClass, - VectorStoreRecordDefinition recordDefinition) { + public VectorStoreRecordMapper getVectorStoreRecordMapper( + Class recordClass, + VectorStoreRecordDefinition recordDefinition) { return PostgreSQLVectorStoreRecordMapper.builder() - .withRecordClass(recordClass) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); } /** diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java index 3d037c70..e3213105 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java @@ -39,20 +39,20 @@ public static void main(String[] args) throws SQLException { .build(); // Get a collection from the vector store - VectorStoreRecordCollection collection = - vectorStore.getCollection("skhotels", - JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(Hotel.class) - .build()); + VectorStoreRecordCollection collection = vectorStore.getCollection( + "skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); // Create the collection if it doesn't exist yet. collection.createCollectionAsync().block(); collection.upsertAsync(new Hotel("1", - "HotelOne", - "Desc for HotelOne", - Collections.emptyList(), Collections.emptyList()), - null) + "HotelOne", + "Desc for HotelOne", + Collections.emptyList(), Collections.emptyList()), + null) .block(); } diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java index 8de0b5aa..77b034bf 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java @@ -32,7 +32,8 @@ public class Book { - public Book() {} + public Book() { + } public Book(String isbn, String title, String author, int pages, List tags, String summary, List summaryEmbedding) { @@ -60,7 +61,7 @@ public Book(String isbn, String title, String author, int pages, @VectorStoreRecordData(isFilterable = true) private List tags; - @VectorStoreRecordData( isFilterable = true, isFullTextSearchable = true ) + @VectorStoreRecordData(isFilterable = true, isFullTextSearchable = true) private String summary; @VectorStoreRecordVector(dimensions = 2) diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index e769bb6f..abb06111 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -194,7 +194,11 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) dataFields.add(VectorStoreRecordDataField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType(), List.class.equals(field.getType()) ? (Class)((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0] : null) + .withFieldType(field.getType(), + List.class.equals(field.getType()) + ? (Class) ((ParameterizedType) field.getGenericType()) + .getActualTypeArguments()[0] + : null) .isFilterable(dataAttribute.isFilterable()) .build()); } @@ -210,7 +214,11 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) vectorFields.add(VectorStoreRecordVectorField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType(), List.class.equals(field.getType()) ? (Class)((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0] : null) + .withFieldType(field.getType(), + List.class.equals(field.getType()) + ? (Class) ((ParameterizedType) field.getGenericType()) + .getActualTypeArguments()[0] + : null) .withDimensions(vectorAttribute.dimensions()) .withIndexKind(vectorAttribute.indexKind()) .withDistanceFunction(vectorAttribute.distanceFunction()) From 9c69a2723fb88ad6d2094833a38b5bb0c313a03e Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:49:34 +0100 Subject: [PATCH 64/64] Fix packageing to include utils in the API --- agents/semantickernel-agents-core/pom.xml | 15 +++++- aiservices/google/pom.xml | 16 +++++++ aiservices/huggingface/pom.xml | 8 ++++ aiservices/openai/pom.xml | 24 ++++++++++ .../semantickernel-data-azureaisearch/pom.xml | 8 ++++ data/semantickernel-data-hsqldb/pom.xml | 23 +++++++-- data/semantickernel-data-jdbc/pom.xml | 11 +++++ data/semantickernel-data-mysql/pom.xml | 23 +++++++-- data/semantickernel-data-oracle/pom.xml | 15 ++++++ data/semantickernel-data-postgres/pom.xml | 13 +++++ data/semantickernel-data-redis/pom.xml | 12 +++++ data/semantickernel-data-sqlite/pom.xml | 13 +++++ pom.xml | 13 +++++ .../booking-agent-m365/pom.xml | 4 ++ .../pom.xml | 4 ++ semantickernel-api-ai-services/pom.xml | 39 +++++++++++++++ .../semantickernel/services/AIService.java | 0 semantickernel-api-builders/pom.xml | 6 +-- semantickernel-api-data/pom.xml | 9 +++- semantickernel-api-exceptions/pom.xml | 1 + .../pom.xml | 6 ++- semantickernel-api/pom.xml | 48 ++++++++++++++++++- semantickernel-api/src/assembly/custom.xml | 22 +++++++++ semantickernel-bom/pom.xml | 8 +++- semantickernel-experimental/pom.xml | 5 ++ 25 files changed, 326 insertions(+), 20 deletions(-) create mode 100644 semantickernel-api-ai-services/pom.xml rename {semantickernel-api-textembedding-services => semantickernel-api-ai-services}/src/main/java/com/microsoft/semantickernel/services/AIService.java (100%) create mode 100644 semantickernel-api/src/assembly/custom.xml diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml index 9d40d75d..cafee140 100644 --- a/agents/semantickernel-agents-core/pom.xml +++ b/agents/semantickernel-agents-core/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 com.microsoft.semantic-kernel @@ -18,6 +19,18 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + \ No newline at end of file diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index a205a60b..390c3bc1 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -34,6 +34,22 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-localization + com.fasterxml.jackson.core diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index aaed6aad..4ce69b95 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -31,6 +31,14 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + com.azure azure-core diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 4a93b679..04b31ca3 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -19,6 +19,30 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-data + provided + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + provided + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + provided + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + + com.azure azure-ai-openai diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index 74dc84bc..038cc038 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -21,6 +21,14 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + com.azure diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml index f755f0f1..42245185 100644 --- a/data/semantickernel-data-hsqldb/pom.xml +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -4,10 +4,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.microsoft.semantic-kernel - semantickernel-parent - 1.4.4-RC2-SNAPSHOT - ../../pom.xml + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml com.microsoft.semantic-kernel @@ -15,7 +15,7 @@ Semantic Kernel HLSQLDB connector Provides a HLSQLDB connector for the Semantic Kernel - + com.microsoft.semantic-kernel semantickernel-api @@ -24,6 +24,19 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + com.fasterxml.jackson.core jackson-databind diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 592cbbe5..f1e62c6a 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -16,6 +16,17 @@ com.microsoft.semantic-kernel semantickernel-api-data + provided + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + provided + + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided org.slf4j diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml index aefd1cd9..899f1f2a 100644 --- a/data/semantickernel-data-mysql/pom.xml +++ b/data/semantickernel-data-mysql/pom.xml @@ -4,10 +4,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.microsoft.semantic-kernel - semantickernel-parent - 1.4.4-RC2-SNAPSHOT - ../../pom.xml + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml com.microsoft.semantic-kernel @@ -15,7 +15,7 @@ Semantic Kernel MySQL connector Provides a MySQL connector for the Semantic Kernel - + com.microsoft.semantic-kernel semantickernel-api @@ -24,6 +24,19 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + com.fasterxml.jackson.core jackson-databind diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index 6514ff0b..aa953870 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -29,6 +29,21 @@ + + com.microsoft.semantic-kernel + semantickernel-api-data + provided + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + provided + + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided + com.microsoft.semantic-kernel semantickernel-data-jdbc diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 73591658..2d01ed81 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -23,6 +23,19 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + com.fasterxml.jackson.core jackson-databind diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index a8e582eb..4af2dc6a 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -17,6 +17,18 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + org.slf4j diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index c8a747a9..64b4a6db 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -24,6 +24,19 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + com.fasterxml.jackson.core jackson-databind diff --git a/pom.xml b/pom.xml index b52f7484..c50eda63 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,7 @@ semantickernel-api-builders semantickernel-api-textembedding-services semantickernel-api-localization + semantickernel-api-ai-services @@ -149,26 +150,31 @@ com.microsoft.semantic-kernel semantickernel-api-builders ${project.version} + provided com.microsoft.semantic-kernel semantickernel-api-data ${project.version} + provided com.microsoft.semantic-kernel semantickernel-api-exceptions ${project.version} + provided com.microsoft.semantic-kernel semantickernel-api-localization ${project.version} + provided com.microsoft.semantic-kernel semantickernel-api-textembedding-services ${project.version} + provided com.microsoft.semantic-kernel.extensions @@ -180,6 +186,13 @@ semantickernel-actionplanner-extension ${project.version} + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + ${project.version} + provided + + com.github.spotbugs diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index f6f8d2a8..d9fd6eee 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -43,6 +43,10 @@ com.microsoft.semantic-kernel semantickernel-aiservices-openai + + com.microsoft.semantic-kernel + semantickernel-api-builders + com.azure diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 10fb9a30..be88deb6 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -29,6 +29,10 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + org.apache.logging.log4j diff --git a/semantickernel-api-ai-services/pom.xml b/semantickernel-api-ai-services/pom.xml new file mode 100644 index 00000000..9b4d76f6 --- /dev/null +++ b/semantickernel-api-ai-services/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + Semantic Kernel AI Services API + Defines the public interface for the Semantic Kernel Services + + + + com.google.code.findbugs + jsr305 + provided + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/AIService.java b/semantickernel-api-ai-services/src/main/java/com/microsoft/semantickernel/services/AIService.java similarity index 100% rename from semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/AIService.java rename to semantickernel-api-ai-services/src/main/java/com/microsoft/semantickernel/services/AIService.java diff --git a/semantickernel-api-builders/pom.xml b/semantickernel-api-builders/pom.xml index f48a2297..5bc0e11b 100644 --- a/semantickernel-api-builders/pom.xml +++ b/semantickernel-api-builders/pom.xml @@ -4,9 +4,9 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.microsoft.semantic-kernel - semantickernel-parent - 1.4.4-RC2-SNAPSHOT + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT com.microsoft.semantic-kernel diff --git a/semantickernel-api-data/pom.xml b/semantickernel-api-data/pom.xml index 895e70e1..822b7c57 100644 --- a/semantickernel-api-data/pom.xml +++ b/semantickernel-api-data/pom.xml @@ -19,14 +19,22 @@ com.microsoft.semantic-kernel semantickernel-api-exceptions + provided com.microsoft.semantic-kernel semantickernel-api-builders + provided com.microsoft.semantic-kernel semantickernel-api-textembedding-services + provided + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + provided com.fasterxml.jackson.core @@ -41,7 +49,6 @@ io.projectreactor reactor-core - 3.4.38 diff --git a/semantickernel-api-exceptions/pom.xml b/semantickernel-api-exceptions/pom.xml index 71d1794b..b459db13 100644 --- a/semantickernel-api-exceptions/pom.xml +++ b/semantickernel-api-exceptions/pom.xml @@ -19,6 +19,7 @@ com.microsoft.semantic-kernel semantickernel-api-localization + provided com.google.code.findbugs diff --git a/semantickernel-api-textembedding-services/pom.xml b/semantickernel-api-textembedding-services/pom.xml index dbc5fa6a..48a524ed 100644 --- a/semantickernel-api-textembedding-services/pom.xml +++ b/semantickernel-api-textembedding-services/pom.xml @@ -19,7 +19,6 @@ io.projectreactor reactor-core - 3.4.38 com.google.code.findbugs @@ -29,7 +28,10 @@ com.github.spotbugs spotbugs-annotations - ${spotbugs.version} + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index d34e5a17..0911e705 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -1,6 +1,7 @@ - + 4.0.0 @@ -15,7 +16,7 @@ Semantic Kernel API Defines the public interface for the Semantic Kernel - + com.microsoft.semantic-kernel semantickernel-api-data @@ -35,6 +36,10 @@ com.microsoft.semantic-kernel semantickernel-api-textembedding-services + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + io.opentelemetry.instrumentation opentelemetry-reactor-3.1 @@ -123,6 +128,45 @@ 1 + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack-dependencies + process-resources + + unpack-dependencies + + + + semantickernel-api-exceptions,semantickernel-api-builders,semantickernel-api-localization,semantickernel-api-textembedding-services,semantickernel-api-ai-services,semantickernel-api-data + + ${project.build.directory}/lib + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + src/assembly/custom.xml + + false + + + + make-assembly + package + + single + + + + diff --git a/semantickernel-api/src/assembly/custom.xml b/semantickernel-api/src/assembly/custom.xml new file mode 100644 index 00000000..48becfff --- /dev/null +++ b/semantickernel-api/src/assembly/custom.xml @@ -0,0 +1,22 @@ + + with-exceptions + + jar + + false + + + ${project.build.outputDirectory} + / + + + ${project.build.directory}/lib + / + + **/** + + + + \ No newline at end of file diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 940093a0..990eb433 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -1,6 +1,7 @@ - + 4.0.0 com.microsoft.semantic-kernel @@ -252,6 +253,11 @@ 4.36.0 true + + io.projectreactor + reactor-core + 3.7.8 + diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 98ce0918..538c9498 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -88,6 +88,11 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided +