From 92a7b82365f7b7c797fd0cd6f8ce3ba29774f17f Mon Sep 17 00:00:00 2001 From: Pradeep Rajendran Date: Fri, 10 May 2024 01:34:14 +0000 Subject: [PATCH 1/4] Adding contentType to document --- .../MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs index a20ede599..f93b625e3 100644 --- a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs +++ b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs @@ -78,7 +78,8 @@ public async Task WriteFileAsync( { "_id", id }, { "documentId", documentId }, { "fileName", fileName }, - { "content", new BsonString(await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false)) } + { "content", new BsonString(await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false)) }, + { "contentType", "text/plain"} }; await this.SaveDocumentAsync(index, id, doc, cancellationToken).ConfigureAwait(false); } @@ -94,6 +95,7 @@ public async Task WriteFileAsync( doc["documentId"] = documentId; doc["fileName"] = fileName; doc["content"] = content; + doc["contentType"] = "text/plain"; await this.SaveDocumentAsync(index, id, doc, cancellationToken).ConfigureAwait(false); } else @@ -105,7 +107,8 @@ public async Task WriteFileAsync( { { "index", index }, { "documentId", documentId }, - { "fileName", fileName } + { "fileName", fileName }, + { "contentType", "application/octet-stream"} } }; From eeeeb0f50f8f69f247b7a528a71ac419d204025d Mon Sep 17 00:00:00 2001 From: Pradeep Rajendran Date: Fri, 10 May 2024 03:59:31 +0000 Subject: [PATCH 2/4] Using standard MIME text --- extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs index f93b625e3..7870f4abb 100644 --- a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs +++ b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs @@ -79,7 +79,7 @@ public async Task WriteFileAsync( { "documentId", documentId }, { "fileName", fileName }, { "content", new BsonString(await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false)) }, - { "contentType", "text/plain"} + { "contentType", Pipeline.MimeTypes.PlainText} }; await this.SaveDocumentAsync(index, id, doc, cancellationToken).ConfigureAwait(false); } @@ -95,7 +95,7 @@ public async Task WriteFileAsync( doc["documentId"] = documentId; doc["fileName"] = fileName; doc["content"] = content; - doc["contentType"] = "text/plain"; + doc["contentType"] = Pipeline.MimeTypes.PlainText; await this.SaveDocumentAsync(index, id, doc, cancellationToken).ConfigureAwait(false); } else @@ -108,7 +108,7 @@ public async Task WriteFileAsync( { "index", index }, { "documentId", documentId }, { "fileName", fileName }, - { "contentType", "application/octet-stream"} + { "contentType", (new Pipeline.MimeTypesDetection()).GetFileType(fileName)} } }; From 7ba64d67770c3f48b1bc191e62cb1e2760d8a48f Mon Sep 17 00:00:00 2001 From: Pradeep Rajendran Date: Fri, 10 May 2024 09:21:08 -0400 Subject: [PATCH 3/4] Using DI for mime type detection --- .../MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs index 7870f4abb..02bc548f2 100644 --- a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs +++ b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.KernelMemory.ContentStorage; +using Microsoft.KernelMemory.Pipeline; using MongoDB.Bson; using MongoDB.Driver; using MongoDB.Driver.GridFS; @@ -15,8 +16,12 @@ namespace Microsoft.KernelMemory.MongoDbAtlas; [Experimental("KMEXP03")] public sealed class MongoDbAtlasStorage : MongoDbAtlasBaseStorage, IContentStorage { - public MongoDbAtlasStorage(MongoDbAtlasConfig config) : base(config) + private readonly IMimeTypeDetection _mimeTypeDetection; + public MongoDbAtlasStorage( + MongoDbAtlasConfig config, + IMimeTypeDetection? mimeTypeDetection = null) : base(config) { + this._mimeTypeDetection = mimeTypeDetection ?? new MimeTypesDetection(); } public Task CreateIndexDirectoryAsync(string index, CancellationToken cancellationToken = default) @@ -108,7 +113,7 @@ public async Task WriteFileAsync( { "index", index }, { "documentId", documentId }, { "fileName", fileName }, - { "contentType", (new Pipeline.MimeTypesDetection()).GetFileType(fileName)} + { "contentType", this._mimeTypeDetection.GetFileType(fileName)} } }; From 819f4024969b00a1981a26fd2e69be30b236c6a5 Mon Sep 17 00:00:00 2001 From: Devis Lucato Date: Fri, 10 May 2024 19:05:36 -0700 Subject: [PATCH 4/4] Fix code style --- .../MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs index 02bc548f2..3bc3307a1 100644 --- a/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs +++ b/extensions/MongoDbAtlas/MongoDbAtlas/MongoDbAtlasStorage.cs @@ -17,6 +17,7 @@ namespace Microsoft.KernelMemory.MongoDbAtlas; public sealed class MongoDbAtlasStorage : MongoDbAtlasBaseStorage, IContentStorage { private readonly IMimeTypeDetection _mimeTypeDetection; + public MongoDbAtlasStorage( MongoDbAtlasConfig config, IMimeTypeDetection? mimeTypeDetection = null) : base(config) @@ -78,13 +79,13 @@ public async Task WriteFileAsync( if (extension == ".txt") { using var reader = new StreamReader(streamContent); - var doc = new BsonDocument() + var doc = new BsonDocument { { "_id", id }, { "documentId", documentId }, { "fileName", fileName }, { "content", new BsonString(await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false)) }, - { "contentType", Pipeline.MimeTypes.PlainText} + { "contentType", MimeTypes.PlainText } }; await this.SaveDocumentAsync(index, id, doc, cancellationToken).ConfigureAwait(false); } @@ -100,7 +101,7 @@ public async Task WriteFileAsync( doc["documentId"] = documentId; doc["fileName"] = fileName; doc["content"] = content; - doc["contentType"] = Pipeline.MimeTypes.PlainText; + doc["contentType"] = MimeTypes.PlainText; await this.SaveDocumentAsync(index, id, doc, cancellationToken).ConfigureAwait(false); } else @@ -113,14 +114,14 @@ public async Task WriteFileAsync( { "index", index }, { "documentId", documentId }, { "fileName", fileName }, - { "contentType", this._mimeTypeDetection.GetFileType(fileName)} + { "contentType", this._mimeTypeDetection.GetFileType(fileName) } } }; // Since the pattern of usage is that you can upload a file for a document id and then update, we need to delete // any existing file with the same id check if the file exists and delete it IAsyncCursor> existingFile = await GetFromBucketByIdAsync(id, bucket, cancellationToken).ConfigureAwait(false); - if (existingFile.Any(cancellationToken)) + if (await existingFile.AnyAsync(cancellationToken).ConfigureAwait(false)) { await bucket.DeleteAsync(id, cancellationToken).ConfigureAwait(false); }