Setting the "pragma journal_size_limit" to zero by default. This should result in the WAL file staying much smaller during continued heavy use of the database.

Author: robbiehanson

Testing/Xcode-mobile/YapDatabase.xcodeproj/project.pbxproj

@@ -888,7 +888,7 @@
 				ORGANIZATIONNAME = "Robbie Hanson";
 				TargetAttributes = {
 					DCAE51E61673FE2600395076 = {
-						DevelopmentTeam = 939WR99F3W;
+						DevelopmentTeam = 4D98PK6HWS;
 					};
 				};
 			};

Testing/Xcode-mobile/YapDatabase/AppDelegate.m

@@ -73,33 +73,74 @@ - (void)yapDatabaseModified:(NSNotification *)notification
 #pragma mark General Debugging
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
+- (NSString *)randomLetters:(NSUInteger)length
+{
+	NSString *alphabet = @"abcdefghijklmnopqrstuvwxyz";
+	NSUInteger alphabetLength = [alphabet length];
+	
+	NSMutableString *result = [NSMutableString stringWithCapacity:length];
+	
+	NSUInteger i;
+	for (i = 0; i < length; i++)
+	{
+		uint32_t randomIndex = arc4random_uniform((uint32_t)alphabetLength);
+		unichar c = [alphabet characterAtIndex:(NSUInteger)randomIndex];
+		
+		[result appendFormat:@"%C", c];
+	}
+	
+	return result;
+}
+
 - (void)debug
 {
 	NSLog(@"Starting debug...");
 
 	NSString *databasePath = [self databasePath:NSStringFromSelector(_cmd)];
+	NSLog(@"databasePath: %@", databasePath);
 
 	[[NSFileManager defaultManager] removeItemAtPath:databasePath error:nil];
 
 	database = [[YapDatabase alloc] initWithPath:databasePath];
-//	database.connectionPoolLifetime = 15;
 
-	databaseConnection = [database newConnection];
+	// Fill up the database with stuff
 
 	[[database newConnection] readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
 
-		[transaction setObject:@"value" forKey:@"key" inCollection:nil];
+		for (int i = 0; i < 1500; i++)
+		{
+			NSString *key = [NSString stringWithFormat:@"%d", i];
+			
+			[transaction setObject:[self randomLetters:100] forKey:key inCollection:nil];
+		}
 	}];
 
-	[NSTimer scheduledTimerWithTimeInterval:30 target:self selector:@selector(debugTimer:) userInfo:nil repeats:YES];
-}
-
-- (void)debugTimer:(NSTimer *)timer
-{
+	// Delete a bunch of stuff (to make more pages in the WAL)
+	
 	[[database newConnection] readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
 
-		[transaction setObject:@"value" forKey:@"key" inCollection:nil];
+		for (int i = 0; i < 1000; i+=2)
+		{
+			NSString *key = [NSString stringWithFormat:@"%d", i];
+			
+			[transaction removeObjectForKey:key inCollection:nil];
+		}
 	}];
+	
+	NSLog(@"database WAL should be fairly big now...");
+	
+	// This next change should reset the WAL file size (to something much smaller)
+	
+	dispatch_time_t when = dispatch_time(DISPATCH_TIME_NOW, (int64_t)(3 * NSEC_PER_SEC));
+	dispatch_after(when, dispatch_get_main_queue(), ^{
+		
+		NSLog(@"This change should reset the WAL file size to something much smaller...");
+		
+		[[database newConnection] readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) {
+			
+			[transaction setObject:@"bar" forKey:@"foo" inCollection:nil];
+		}];
+	});
 }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

YapDatabase/YapDatabase.m

@@ -572,6 +572,21 @@ - (BOOL)configureDatabase:(BOOL)isNewDatabaseFile
 		// This isn't critical, so we can continue.
 	}
 
+	// Set journal_size_imit.
+	//
+	// We only need to do set this pragma for THIS connection,
+	// because it is the only connection that performs checkpoints.
+	
+	NSString *stmt =
+	  [NSString stringWithFormat:@"PRAGMA journal_size_limit = %ld;", (long)options.pragmaJournalSizeLimit];
+	
+	status = sqlite3_exec(db, [stmt UTF8String], NULL, NULL, NULL);
+	if (status != SQLITE_OK)
+	{
+		YDBLogError(@"Error setting PRAGMA journal_size_limit: %d %s", status, sqlite3_errmsg(db));
+		// This isn't critical, so we can continue.
+	}
+	
 	// Disable autocheckpointing.
 	//
 	// YapDatabase has its own optimized checkpointing algorithm built-in.
@@ -2345,6 +2360,8 @@ - (void)noteCommittedChanges:(NSDictionary *)changeset fromConnection:(YapDataba
 **/
 - (void)asyncCheckpoint:(uint64_t)maxCheckpointableSnapshot
 {
+	static BOOL const PRINT_WAL_SIZE = NO;
+	
 	__weak YapDatabase *weakSelf = self;
 
 	dispatch_async(checkpointQueue, ^{ @autoreleasepool {
@@ -2354,6 +2371,18 @@ - (void)asyncCheckpoint:(uint64_t)maxCheckpointableSnapshot
 
 		YDBLogVerbose(@"Checkpointing up to snapshot %llu", maxCheckpointableSnapshot);
 
+		if (YDB_LOG_VERBOSE && PRINT_WAL_SIZE)
+		{
+			NSString *walFilePath = [strongSelf.databasePath stringByAppendingString:@"-wal"];
+			
+			NSDictionary *walAttr = [[NSFileManager defaultManager] attributesOfItemAtPath:walFilePath error:NULL];
+			unsigned long long walFileSize = [walAttr fileSize];
+			
+			YDBLogVerbose(@"Pre-checkpoint file size: %@",
+			  [NSByteCountFormatter stringFromByteCount:(long long)walFileSize
+			                                 countStyle:NSByteCountFormatterCountStyleFile]);
+		}
+		
 		// We're ready to checkpoint more frames.
 		//
 		// So we're going to execute a passive checkpoint.
@@ -2417,6 +2446,18 @@ - (void)asyncCheckpoint:(uint64_t)maxCheckpointableSnapshot
 				}
 			});
 		}
+		
+		if (YDB_LOG_VERBOSE && PRINT_WAL_SIZE)
+		{
+			NSString *walFilePath = [strongSelf.databasePath stringByAppendingString:@"-wal"];
+			
+			NSDictionary *walAttr = [[NSFileManager defaultManager] attributesOfItemAtPath:walFilePath error:NULL];
+			unsigned long long walFileSize = [walAttr fileSize];
+			
+			YDBLogVerbose(@"Post-checkpoint file size: %@",
+			  [NSByteCountFormatter stringFromByteCount:(long long)walFileSize
+			                                 countStyle:NSByteCountFormatterCountStyleFile]);
+		}
 	}});
 }
 

YapDatabase/YapDatabaseOptions.h

@@ -63,6 +63,19 @@ typedef NSString* (^YapDatabaseOptionsPassphraseBlock)(void);
 **/
 @property (nonatomic, assign, readwrite) YapDatabasePragmaSynchronous pragmaSynchronous;
 
+/**
+ * Allows you to configure the sqlite "PRAGMA journal_size_limit" option.
+ * 
+ * For more information, see the sqlite docs:
+ * http://www.sqlite.org/pragma.html#pragma_journal_size_limit
+ * 
+ * Note that YapDatabase uses sqlite in WAL mode.
+ * 
+ * The default value is zero,
+ * meaning that every checkpoint will reduce the WAL file to its minimum size (if possible).
+**/
+@property (nonatomic, assign, readwrite) NSInteger pragmaJournalSizeLimit;
+
 #ifdef SQLITE_HAS_CODEC
 /**
  * Set a block here that returns the passphrase for the SQLCipher

YapDatabase/YapDatabaseOptions.m

@@ -16,13 +16,15 @@ @implementation YapDatabaseOptions
 
 @synthesize corruptAction = corruptAction;
 @synthesize pragmaSynchronous = pragmaSynchronous;
+@synthesize pragmaJournalSizeLimit = pragmaJournalSizeLimit;
 
 - (id)init
 {
 	if ((self = [super init]))
 	{
 		corruptAction = YapDatabaseCorruptAction_Rename;
 		pragmaSynchronous = YapDatabasePragmaSynchronous_Full;
+		pragmaJournalSizeLimit = 0;
 	}
 	return self;
 }
@@ -32,6 +34,7 @@ - (id)copyWithZone:(NSZone *)zone
 	YapDatabaseOptions *copy = [[[self class] alloc] init];
 	copy->corruptAction = corruptAction;
 	copy->pragmaSynchronous = pragmaSynchronous;
+	copy->pragmaJournalSizeLimit = pragmaJournalSizeLimit;
 #ifdef SQLITE_HAS_CODEC
     copy.passphraseBlock = _passphraseBlock;
 #endif