Fix Bug #3355: Fix that long running big upload (250GB+) fails because of an expired access token (#3361)

abraunegg · web-flow · commit ea7c3abd2d3b · 2025-07-03T17:21:16.000+10:00
* Revert back to v2.5.5 performSessionFileUpload() and apply minimal change for upload session offset handling to prevent desynchronisation on large files
* Add specific 403 handler for when the upload session URL itself expires
* Add 'file_fragment_size'
* Clean up debug logging output
* Add 'tempauth' to spelling words
* Update documentation URL's
* Ensure that on each fragment upload, whilst the application is using the 'tempauth' for session upload, the global OAuth2 token needs to be checked for validity and refreshed if required
* Add limit check for 'file_fragment_size' option
* Add to default 'config' file
* Update documentation for 'file_fragment_size'
* Add 'file_fragment_size' to --display-config output
* Add --file-fragment-size option to enable use via Docker option
* Add to manpage
* Update Docker entrypoint
* Update Docker | Podman documentation
* Update logging output to include connection method to URL
* Update Upload Session URL expiry update to include UTC and LocalTime values
* Update comment which was dropped / missed
* Clarify that this is the OAuth2 Access Token
* Clarify that the expiry timestamp is localTime
* Update PR with dynamic use of fragment size if fileSize &gt; 100MiB
* Enforce multiple 320KiB for fragment size to align to Microsoft documentation
* Fix Docker entrypoint and confirm working for ONEDRIVE_FILE_FRAGMENT_SIZE
* Change 'defaultMaxFileFragmentSize' to 60
* Revise fragmentSize calculation to be as close to 60 MiB as possible without breaching Microsoft documented threshold
diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt
@@ -445,6 +445,7 @@ systemdsystemunitdir
 systemduserunitdir
 tbh
 tdcockers
+tempauth
 templ
 testbuild
 Thh
diff --git a/config b/config
@@ -76,6 +76,9 @@
 ## This setting controls the application logging all actions to a separate file.
 #enable_logging = "false"
 
+## This setting controls the file fragment size when uploading large files to Microsoft OneDrive. 
+#file_fragment_size = "10"
+
 ## This setting controls the application HTTP protocol version, downgrading to HTTP/1.1 when enabled.
 #force_http_11 = "false"
 
diff --git a/contrib/docker/entrypoint.sh b/contrib/docker/entrypoint.sh
@@ -174,6 +174,13 @@ if [ "${ONEDRIVE_SYNC_SHARED_FILES:=0}" == "1" ]; then
 	ARGS=(--sync-shared-files ${ARGS[@]})
 fi
 
+# Tell client to use a different value for file fragment size for large file uploads
+if [ -n "${ONEDRIVE_FILE_FRAGMENT_SIZE:=""}" ]; then
+	echo "# We are specifying the file fragment size for large file uploads (in MB)"
+	echo "# Adding --file-fragment-size ARG"
+	ARGS=(--file-fragment-size ${ONEDRIVE_FILE_FRAGMENT_SIZE} ${ARGS[@]})
+fi
+
 if [ ${#} -gt 0 ]; then
 	ARGS=("${@}")
 fi
diff --git a/docs/application-config-options.md b/docs/application-config-options.md
@@ -29,6 +29,7 @@ Before reading this document, please ensure you are running application version
   - [drive_id](#drive_id)
   - [dry_run](#dry_run)
   - [enable_logging](#enable_logging)
+  - [file_fragment_size](#file_fragment_size)
   - [force_http_11](#force_http_11)
   - [force_session_upload](#force_session_upload)
   - [inotify_delay](#inotify_delay)
@@ -412,6 +413,20 @@ _**CLI Option Use:**_ `--enable-logging`
 > [!IMPORTANT]
 > Additional configuration is potentially required to configure the default log directory. Refer to the [Enabling the Client Activity Log](./usage.md#enabling-the-client-activity-log) section in usage.md for details
 
+### file_fragment_size
+_**Description:**_ This option controls the fragment size when uploading large files to Microsoft OneDrive. The value specified is in MB.
+
+_**Value Type:**_ Integer
+
+_**Default Value:**_ 10
+
+_**Maximum Value:**_ 60
+
+_**Config Example:**_ `file_fragment_size = "25"`
+
+_**CLI Option Use:**_ `--file-fragment-size = '25'`
+
+
 ### force_http_11
 _**Description:**_ This setting controls the application HTTP protocol version. By default, the application will use libcurl defaults for which HTTP protocol version will be used to interact with Microsoft OneDrive. Use this setting to downgrade libcurl to only use HTTP/1.1.
 
@@ -871,6 +886,10 @@ _**Config Example:**_ `skip_size = "50"`
 
 _**CLI Option Use:**_ `--skip-size '50'`
 
+> [!NOTE]
+> This option is considered a 'Client Side Filtering Rule' and if configured, is utilised for all sync operations. After changing this option, you will be required to perform a resync.
+
+
 ### skip_symlinks
 _**Description:**_ This configuration option controls whether the application will skip all symbolic links when performing sync operations. Microsoft OneDrive has no concept or understanding of symbolic links, and attempting to upload a symbolic link to Microsoft OneDrive generates a platform API error. All data (files and folders) that are uploaded to OneDrive must be whole files or actual directories.
 
diff --git a/docs/docker.md b/docs/docker.md
@@ -290,6 +290,7 @@ docker run $firstRun --restart unless-stopped --name onedrive -v onedrive_conf:/
 | <B>ONEDRIVE_SYNC_SHARED_FILES</B> | Controls "--sync-shared-files" option. Default is 0 | 1 |
 | <B>ONEDRIVE_RUNAS_ROOT</B> | Controls if the Docker container should be run as the 'root' user instead of 'onedrive' user. Default is 0 | 1 |
 | <B>ONEDRIVE_SYNC_ONCE</B> | Controls if the Docker container should be run in Standalone Mode. It will use Monitor Mode otherwise. Default is 0 | 1 |
+| <B>ONEDRIVE_FILE_FRAGMENT_SIZE</B> | Controls the fragment size when uploading large files to Microsoft OneDrive. The value specified is in MB. Default is 10, Limit is 60 | 25 |
 
 ### Environment Variables Usage Examples
 **Verbose Output:**
diff --git a/docs/podman.md b/docs/podman.md
@@ -305,6 +305,7 @@ podman run -it --name onedrive_work --user "${ONEDRIVE_UID}:${ONEDRIVE_GID}" \
 | <B>ONEDRIVE_SYNC_SHARED_FILES</B> | Controls "--sync-shared-files" option. Default is 0 | 1 |
 | <B>ONEDRIVE_RUNAS_ROOT</B> | Controls if the Docker container should be run as the 'root' user instead of 'onedrive' user. Default is 0 | 1 |
 | <B>ONEDRIVE_SYNC_ONCE</B> | Controls if the Docker container should be run in Standalone Mode. It will use Monitor Mode otherwise. Default is 0 | 1 |
+| <B>ONEDRIVE_FILE_FRAGMENT_SIZE</B> | Controls the fragment size when uploading large files to Microsoft OneDrive. The value specified is in MB. Default is 10, Limit is 60 | 25 |
 
 ### Environment Variables Usage Examples
 **Verbose Output:**
diff --git a/onedrive.1.in b/onedrive.1.in
@@ -204,6 +204,10 @@ Perform a trial sync with no changes made.
 \fB\-\-enable-logging\fR
 Enable client activity to a separate log file.
 
+.TP
+\fB\-\-file-fragment-size\fR
+Specify the file fragment size for large file uploads (in MB).
+
 .TP
 \fB\-\-force\fR
 Force the deletion of data when a 'big delete' is detected.
diff --git a/src/config.d b/src/config.d
@@ -46,6 +46,9 @@ class ApplicationConfig {
 	immutable string defaultConfigDirName = "~/.config/onedrive";
 	// - Default 'OneDrive Business Shared Files' Folder Name
 	immutable string defaultBusinessSharedFilesDirectoryName = "Files Shared With Me";
+	// - Default file fragment size for uploads
+	immutable long defaultFileFragmentSize = 10;
+	immutable long defaultMaxFileFragmentSize = 60;
 	
 	// Microsoft Requirements 
 	// - Default Application ID (abraunegg)
@@ -288,6 +291,8 @@ class ApplicationConfig {
 		longValues["rate_limit"] = 0;
 		// - To ensure we do not fill up the load disk, how much disk space should be reserved by default
 		longValues["space_reservation"] = 50 * 2^^20; // 50 MB as Bytes
+		// - How large should our file fragments be when uploading as an 'upload session' ?
+		longValues["file_fragment_size"] = defaultFileFragmentSize; // whole number, treated as MB, will be converted to bytes within performSessionFileUpload(). Default is 10.
 		
 		// HTTPS & CURL Operation Settings
 		// - Maximum time an operation is allowed to take
@@ -1040,6 +1045,20 @@ class ApplicationConfig {
 						tempValue = 0;
 					}
 					setValueLong("skip_size", tempValue);
+				} else if (key == "file_fragment_size") {
+					ulong tempValue = thisConfigValue;
+					// If set, this must be greater than the default, but also aligning to Microsoft upper limit of 60 MiB
+					// Enforce lower bound (must be greater than default)
+					if (tempValue < defaultFileFragmentSize) {
+						addLogEntry("Invalid value for key in config file (too low) - using default value: " ~ key);
+						tempValue = defaultFileFragmentSize;
+					}
+					// Enforce upper bound (safe maximum)
+					else if (tempValue > defaultMaxFileFragmentSize) {
+						addLogEntry("Invalid value for key in config file (too high) - using maximum safe value: " ~ key);
+						tempValue = defaultMaxFileFragmentSize;
+					}
+					setValueLong("file_fragment_size", tempValue);
 				}
 			} else {
 				addLogEntry("Unknown key in config file: " ~ key);
@@ -1138,25 +1157,25 @@ class ApplicationConfig {
 				std.getopt.config.bundling,
 				std.getopt.config.caseSensitive,
 				"auth-files",
-					"Perform authentication not via interactive dialog but via files read/writes to these files.",
+					"Perform authentication not via interactive dialog but via files read/writes to these files",
 					&stringValues["auth_files"],
 				"auth-response",
-					"Perform authentication not via interactive dialog but via providing the response url directly.",
+					"Perform authentication not via interactive dialog but via providing the response url directly",
 					&stringValues["auth_response"],
 				"check-for-nomount",
-					"Check for the presence of .nosync in the syncdir root. If found, do not perform sync.",
+					"Check for the presence of .nosync in the syncdir root. If found, do not perform sync",
 					&boolValues["check_nomount"],
 				"check-for-nosync",
-					"Check for the presence of .nosync in each directory. If found, skip directory from sync.",
+					"Check for the presence of .nosync in each directory. If found, skip directory from sync",
 					&boolValues["check_nosync"],
 				"classify-as-big-delete",
 					"Number of children in a path that is locally removed which will be classified as a 'big data delete'",
 					&longValues["classify_as_big_delete"],
 				"cleanup-local-files",
-					"Cleanup additional local files when using --download-only. This will remove local data.",
+					"Cleanup additional local files when using --download-only. This will remove local data",
 					&boolValues["cleanup_local_files"],	
 				"create-directory",
-					"Create a directory on OneDrive - no sync will be performed.",
+					"Create a directory on OneDrive - no sync will be performed",
 					&stringValues["create_directory"],
 				"create-share-link",
 					"Create a shareable link for an existing file on OneDrive",
@@ -1165,10 +1184,10 @@ class ApplicationConfig {
 					"Debug OneDrive HTTPS communication.",
 					&boolValues["debug_https"],
 				"destination-directory",
-					"Destination directory for renamed or move on OneDrive - no sync will be performed.",
+					"Destination directory for renamed or move on OneDrive - no sync will be performed",
 					&stringValues["destination_directory"],
 				"disable-notifications",
-					"Do not use desktop notifications in monitor mode.",
+					"Do not use desktop notifications in monitor mode",
 					&boolValues["disable_notifications"],
 				"disable-download-validation",
 					"Disable download validation when downloading from OneDrive",
@@ -1177,26 +1196,29 @@ class ApplicationConfig {
 					"Disable upload validation when uploading to OneDrive",
 					&boolValues["disable_upload_validation"],
 				"display-config",
-					"Display what options the client will use as currently configured - no sync will be performed.",
+					"Display what options the client will use as currently configured - no sync will be performed",
 					&boolValues["display_config"],
 				"display-running-config",
-					"Display what options the client has been configured to use on application startup.",
+					"Display what options the client has been configured to use on application startup",
 					&boolValues["display_running_config"],
 				"display-sync-status",
-					"Display the sync status of the client - no sync will be performed.",
+					"Display the sync status of the client - no sync will be performed",
 					&boolValues["display_sync_status"],
 				"display-quota",
-					"Display the quota status of the client - no sync will be performed.",
+					"Display the quota status of the client - no sync will be performed",
 					&boolValues["display_quota"],
 				"download-only",
-					"Replicate the OneDrive online state locally, by only downloading changes from OneDrive. Do not upload local changes to OneDrive.",
+					"Replicate the OneDrive online state locally, by only downloading changes from OneDrive. Do not upload local changes to OneDrive",
 					&boolValues["download_only"],
 				"dry-run",
 					"Perform a trial sync with no changes made",
 					&boolValues["dry_run"],
 				"enable-logging",
 					"Enable client activity to a separate log file",
 					&boolValues["enable_logging"],
+				"file-fragment-size",
+					"Specify the file fragment size for large file uploads (in MB)",
+					&longValues["file_fragment_size"],
 				"force-http-11",
 					"Force the use of HTTP 1.1 for all operations",
 					&boolValues["force_http_11"],
@@ -1222,10 +1244,10 @@ class ApplicationConfig {
 					"Sync OneDrive Business Shared Files to the local filesystem",
 					&boolValues["sync_business_shared_files"],
 				"local-first",
-					"Synchronize from the local directory source first, before downloading changes from OneDrive.",
+					"Synchronize from the local directory source first, before downloading changes from OneDrive",
 					&boolValues["local_first"],
 				"log-dir",
-					"Directory where logging output is saved to, needs to end with a slash.",
+					"Directory where logging output is saved to, needs to end with a slash",
 					&stringValues["log_dir"],
 				"logout",
 					"Logout the current user",
@@ -1237,7 +1259,7 @@ class ApplicationConfig {
 					"Keep monitoring for local and remote changes",
 					&boolValues["monitor"],
 				"monitor-interval",
-					"Number of seconds by which each sync operation is undertaken when idle under monitor mode.",
+					"Number of seconds by which each sync operation is undertaken when idle under monitor mode",
 					&longValues["monitor_interval"],
 				"monitor-fullscan-frequency",
 					"Number of sync runs before performing a full local scan of the synced directory",
@@ -1261,13 +1283,13 @@ class ApplicationConfig {
 					"Approve the use of performing a --resync action",
 					&boolValues["resync_auth"],
 				"remove-directory",
-					"Remove a directory on OneDrive - no sync will be performed.",
+					"Remove a directory on OneDrive - no sync will be performed",
 					&stringValues["remove_directory"],
 				"remove-source-files",
 					"Remove source file after successful transfer to OneDrive when using --upload-only",
 					&boolValues["remove_source_files"],
 				"single-directory",
-					"Specify a single local directory within the OneDrive root to sync.",
+					"Specify a single local directory within the OneDrive root to sync",
 					&stringValues["single_directory"],
 				"skip-dot-files",
 					"Skip dot files and folders from syncing",
@@ -1288,7 +1310,7 @@ class ApplicationConfig {
 					"Skip syncing of symlinks",
 					&boolValues["skip_symlinks"],
 				"source-directory",
-					"Source directory to rename or move on OneDrive - no sync will be performed.",
+					"Source directory to rename or move on OneDrive - no sync will be performed",
 					&stringValues["source_directory"],
 				"space-reservation",
 					"The amount of disk space to reserve (in MB) to avoid 100% disk space utilisation",
@@ -1306,10 +1328,10 @@ class ApplicationConfig {
 					"Perform a synchronisation with Microsoft OneDrive (DEPRECATED)",
 					&boolValues["synchronize"],
 				"sync-root-files",
-					"Sync all files in sync_dir root when using sync_list.",
+					"Sync all files in sync_dir root when using sync_list",
 					&boolValues["sync_root_files"],
 				"upload-only",
-					"Replicate the locally configured sync_dir state to OneDrive, by only uploading local changes to OneDrive. Do not download changes from OneDrive.",
+					"Replicate the locally configured sync_dir state to OneDrive, by only uploading local changes to OneDrive. Do not download changes from OneDrive",
 					&boolValues["upload_only"],
 				"confdir",
 					"Set the directory used to store the configuration files",
@@ -1325,7 +1347,7 @@ class ApplicationConfig {
 					&boolValues["with_editing_perms"]
 			);
 			
-			// Was --syncdir used?
+			// Was --syncdir specified?
 			if (!getValueString("sync_dir_cli").empty) {
 				// Build the line we need to update and/or write out
 				string newConfigOptionSyncDirLine = "sync_dir = \"" ~ getValueString("sync_dir_cli") ~ "\"";
@@ -1411,12 +1433,24 @@ class ApplicationConfig {
 				setValueString("sync_dir", getValueString("sync_dir_cli"));
 			}
 			
-			// was --monitor-interval used and now set to a value below minimum requirement?
+			// Was --monitor-interval specified and now set to a value below minimum requirement?
 			if (getValueLong("monitor_interval") < 300 ) {
 				addLogEntry("Invalid value for --monitor-interval - using default value: 300");
 				setValueLong("monitor_interval", 300);
 			}
 			
+			// Was --file-fragment-size specified and now set to a value below or above maximum?
+			// Enforce lower bound (must be greater than default) for 'file_fragment_size'
+			if (getValueLong("file_fragment_size") < defaultFileFragmentSize) {
+				addLogEntry("Invalid value for --file-fragment-size (too low) - using default value: " ~ to!string(defaultFileFragmentSize));
+				setValueLong("file_fragment_size", defaultFileFragmentSize);
+			}
+			// Enforce upper bound (safe maximum) for 'file_fragment_size'
+			if (getValueLong("file_fragment_size") > defaultMaxFileFragmentSize) {
+				addLogEntry("Invalid value for --file-fragment-size (too high) - using maximum safe value: " ~ to!string(defaultMaxFileFragmentSize));
+				setValueLong("file_fragment_size", defaultMaxFileFragmentSize);
+			}
+			
 			// Was --auth-files used?
 			if (!getValueString("auth_files").empty) {
 				// --auth-files used, need to validate that '~' was not used as a path identifier, and if yes, perform the correct expansion
@@ -1597,6 +1631,7 @@ class ApplicationConfig {
 		addLogEntry("Config option 'inotify_delay'                = " ~ to!string(getValueLong("inotify_delay")));
 		addLogEntry("Config option 'display_transfer_metrics'     = " ~ to!string(getValueBool("display_transfer_metrics")));
 		addLogEntry("Config option 'force_session_upload'         = " ~ to!string(getValueBool("force_session_upload")));
+		addLogEntry("Config option 'file_fragment_size'           = " ~ to!string(getValueLong("file_fragment_size")));
 		
 		// data integrity
 		addLogEntry("Config option 'classify_as_big_delete'       = " ~ to!string(getValueLong("classify_as_big_delete")));
diff --git a/src/onedrive.d b/src/onedrive.d
diff --git a/src/sync.d b/src/sync.d
diff --git a/src/util.d b/src/util.d
