Update curl client

Author: dluc

README.md

@@ -1,11 +1,12 @@
 # Semantic Memory
 
-**Semantic Memory** is an open-source library and service specializing in the
-efficient indexing of datasets through custom continuous data pipelines.
+**Semantic Memory** is an open-source library and [service](dotnet/Service)
+specialized in the efficient indexing of datasets through custom continuous data
+pipelines.
 
 ![image](https://github.com/microsoft/semantic-memory/assets/371009/31894afa-d19e-4e9b-8d0f-cb889bf5c77f)
 
-Utilizing advanced embeddings and LLMs, the system enables natural language
+Utilizing advanced embeddings and LLMs, the system enables Natural Language
 querying for obtaining answers from the indexed data, complete with citations
 and links to the original sources.
 
@@ -18,19 +19,23 @@ Semantic Memory enhances data-driven features in applications built using SK.
 > ℹ️ **NOTE**: the documentation below is work in progress, will evolve quickly
 > as is not fully functional yet.
 
-# Examples
+# Importing memory in serverless mode
 
-## Importing memory, locally, without deployments
+Semantic Memory works and scales at best when running as a service, allowing to
+ingest thousands of documents and information without blocking your app.
 
-Importing documents into your Semantic Memory can be as simple as this:
+However, you can use Semantic Memory also serverless, embedding the `MemoryPipelineClient`
+in your app. Importing documents into your Semantic Memory can be as simple as this:
 
 ```csharp
 var memory = new MemoryPipelineClient();
 
+# Import a file (default user)
 await memory.ImportFileAsync("meeting-transcript.docx");
 
+# Import a file specifying a User and Tags
 await memory.ImportFileAsync("business-plan.docx",
-    new DocumentDetails("file1", "user0022")
+    new DocumentDetails("file1", "user@some.email")
         .AddTag("collection", "business")
         .AddTag("collection", "plans")
         .AddTag("type", "doc"));
@@ -41,26 +46,28 @@ Asking questions
 ```csharp
 string answer1 = await memory.AskAsync("How many people attended the meeting?");
 
-string answer2 = await memory.AskAsync("what's the project timeline?", "user0022");
+string answer2 = await memory.AskAsync("what's the project timeline?", "user@some.email");
 ```
 
 The code leverages the default documents ingestion pipeline:
 
-1. Extract text
-2. Partition the text in small chunks
-3. Extract embedding
-4. Save embedding into a vector index
+1. Extract text: recognize the file format and extract the information
+2. Partition the text in small chunks, to optimize search
+3. Extract embedding using an LLM embedding generator
+4. Save embedding into a vector index such as
+   [Azure Cognitive Search](https://learn.microsoft.com/en-us/azure/search/vector-search-overview),
+   [Qdrant](https://qdrant.tech/) or other DBs.
 
-Data is also organized by users, protecting information and allowing to
-organize private information. And memories can be labelled and organized
-using **Tags**.
+Documents are organized by users, safeguarding their private information.
+Furthermore, memories can be categorized and structured using **tags**, enabling
+efficient search and retrieval through faceted navigation.
 
 ## Import memory using Semantic Memory Web Service
 
 Depending on your scenarios, you might want to run all the code **locally
 inside your process, or remotely through an asynchronous service.**
 
-If you're importing small files, and need only C# or Python, and can block
+If you're importing small files, and need only C# or only Python, and can block
 the process during the import, local-in-process execution can be fine, using
 the **MemoryPipelineClient** seen above.
 

dotnet/CoreLib/WebService/UploadRequest.cs

@@ -67,12 +67,30 @@ public class UploadRequest
         {
             if (key == documentIdField || key == userIdField || !form.TryGetValue(key, out StringValues values)) { continue; }
 
-            foreach (var x in values)
+            ValidateTagName(key);
+            foreach (string? x in values)
             {
                 result.Tags.Add(key, x);
             }
         }
 
         return (result, true, string.Empty);
     }
+
+    private static void ValidateTagName(string key)
+    {
+        if (key.Contains("="))
+        {
+            throw new SemanticMemoryException("A tag name cannot contain the '=' symbol");
+        }
+
+        if (key is Constants.ReservedUserIdTag
+            or Constants.ReservedDocIdTag
+            or Constants.ReservedFileIdTag
+            or Constants.ReservedFilePartitionTag
+            or Constants.ReservedFileTypeTag)
+        {
+            throw new SemanticMemoryException($"The tag name '{key}' is reserved for internal use.");
+        }
+    }
 }

samples/curl/example.sh

@@ -7,5 +7,5 @@ cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/"
 ../../curl/upload-file.sh -f test.pdf \
                           -s http://127.0.0.1:9001/upload \
                           -u curlUser \
-                          -c curlDataCollection \
+                          -t "type=test" \
                           -i curlExample01

tools/upload-file.sh

@@ -13,19 +13,21 @@ Help for Bash script
 
 Usage:
 
-    ./upload-file.sh -f <file path> -u <id> -c <list> -i <id> -s <url>
+    ./upload-file.sh -s <url> -f <file path> -u <id> [-i <id>] [-t <tag1> -t <tag2> -t <tag3> (...)]
+
+    -s web service URL     (required) Semantic Memory web service URL.
+    -f file path           (required) Path to the document to upload.
+    -u userId              (required) User ID.
+
+    -i document ID         (optional) Unique identifier for the document uploaded.
+    -t "key=value"         (optional) Key-Value tag. Multiple tags and values per tag can be set.
 
-    -f file path           Path to the document to upload.
-    -u userId              User ID.
-    -c "coll1 coll2 .."    List of collection IDs separated by a space.
-    -i document ID         Unique identifier for the document uploaded.
-    -s web service URL     Semantic Memory web service URL.
     -h                     Print this help content.
 
 
 Example:
 
-    ./upload-file.sh -f myFile.pdf -u me -c "notes meetings" -i "bash test" -s http://127.0.0.1:9001/upload
+    ./upload-file.sh -f myFile.pdf -u me -t "type=notes" -t "type=test" -i "bash test" -s http://127.0.0.1:9001/upload
 
 
 For more information visit https://github.com/microsoft/semantic-memory
@@ -36,6 +38,10 @@ _EOF_
 readParameters() {
   while [ "$1" != "" ]; do
     case $1 in
+    -s)
+      shift
+      SERVICE_URL=$1
+      ;;
     -f)
       shift
       FILENAME=$1
@@ -44,17 +50,13 @@ readParameters() {
       shift
       USER_ID=$1
       ;;
-    -c)
-      shift
-      COLLECTIONS=$1
-      ;;
     -i)
       shift
       DOCUMENT_ID=$1
       ;;
-    -s)
+    -t)
       shift
-      SERVICE_URL=$1
+      TAGS="$TAGS $1"
       ;;
     *)
       help
@@ -83,27 +85,19 @@ validatePrameters() {
     help
     exit 1
   fi
-  if [ -z "$USER_ID" ]; then
-    echo "Please specify the user ID"
+  if [ -z "$SERVICE_URL" ]; then
+    echo "Please specify the web service URL"
     exit 2
   fi
-  if [ -z "$COLLECTIONS" ]; then
-    echo "Please specify the list of collection IDs"
+  if [ -z "$USER_ID" ]; then
+    echo "Please specify the user ID"
     exit 3
   fi
-  if [ -z "$DOCUMENT_ID" ]; then
-    echo "Please specify a unique document ID"
-    exit 4
-  fi
-  if [ -z "$SERVICE_URL" ]; then
-    echo "Please specify the web service URL"
-    exit 5
-  fi
 }
 
 # Remove variables and functions from the environment, in case the script was sourced
 cleanupEnv() {
-  unset FILENAME USER_ID COLLECTIONS DOCUMENT_ID SERVICE_URL
+  unset FILENAME SERVICE_URL USER_ID TAGS DOCUMENT_ID
   unset -f help readParameters validatePrameters cleanupEnv exitScript
 }
 
@@ -116,17 +110,17 @@ exitScript() {
 readParameters "$@"
 validatePrameters
 
-# Handle list of collection IDs
-COLLECTIONS_FIELD=""
-for x in $COLLECTIONS; do
-  COLLECTIONS_FIELD="${COLLECTIONS_FIELD} -F collections=\"${x}\""
+# Handle list of tags
+TAGS_FIELD=""
+for x in $TAGS; do
+  TAGS_FIELD="${TAGS_FIELD} -F ${x}"
 done
 
 # Send HTTP request using curl
-#set -x
+set -x
 curl -v \
   -F 'file1=@"'"${FILENAME}"'"' \
-  -F 'user="'"${USER_ID}"'"' \
+  -F 'userId="'"${USER_ID}"'"' \
   -F 'documentId="'"${DOCUMENT_ID}"'"' \
-  $COLLECTIONS_FIELD \
+  $TAGS_FIELD \
   $SERVICE_URL