* Updated map cache system from jA

git-svn-id: https://svn.code.sf.net/p/rathena/svn/branches/stable@824 54d463be-8e91-2dee-dedb-b68131a5f0ec

Changelog.txt

@@ -1,5 +1,7 @@
 Date	Added
 12/27
+        * Updated map cache system from jA 1087 - change read_map_from_bitmap to 2 to
+          enable zlib compression [celest]
 	* don't delete alive_timers of -1  (SVN 823) [MouseJstr]
 	* switched malloc,calloc,realloc to aMalloc, aCalloc, aRealloc
 	  so support the use of a garbage collector (SVN 821) [MouseJstr]

conf-tmpl/map_athena.conf

@@ -45,7 +45,10 @@ char_port: 6121
 map_port: 5121
 
 //Preferred map loading method
-//0: Read directly from grf, 1: Read from bitmap file
+//0: Read directly from grf, 1: Read from bitmap file (without compression)
+//2: Read from bitmap file (with compression). It is possible to reduce the map cache
+// to 1MB for 400+ maps with compression enabled. If all maps are already loaded in the
+// cache, Athena can boot without reading the grf files.
 read_map_from_bitmap: 1
 //
 //Where is the bitmap file stored?

src/common/grfio.c

@@ -33,6 +33,30 @@
 #include "showmsg.h"
 #include "malloc.h"
 
+#ifdef _WIN32
+	#include <windows.h>
+	#include "zlib_win32.h"
+	HINSTANCE zlib_dll;
+	#define zlib_inflateInit(strm) zlib_inflateInit_((strm),ZLIB_VERSION, sizeof(z_stream))
+	#define zlib_deflateInit(strm, level) zlib_deflateInit_((strm),(level),ZLIB_VERSION,sizeof(z_stream))
+
+	int (WINAPI* zlib_inflateInit_) (z_streamp strm, const char *version, int stream_size);
+	int (WINAPI* zlib_inflate) (z_streamp strm, int flush);
+	int (WINAPI* zlib_inflateEnd) (z_streamp strm);
+
+	int (WINAPI* zlib_deflateInit_) (z_streamp strm, int level, const char *version, int stream_size);
+	int (WINAPI* zlib_deflate) (z_streamp strm, int flush);
+	int (WINAPI* zlib_deflateEnd) (z_streamp strm);
+#else
+	#include <zlib.h>
+	#define zlib_inflateInit inflateInit
+	#define zlib_inflate     inflate
+	#define zlib_inflateEnd  inflateEnd
+	#define zlib_deflateInit deflateInit
+	#define zlib_deflate     deflate
+	#define zlib_deflateEnd  deflateEnd
+#endif
+
 #ifdef MEMWATCH
 #include "memwatch.h"
 #endif
@@ -260,7 +284,7 @@ static void decode_des_etc(BYTE *buf,int len,int type,int cycle)
  *	Grf data decode sub : zip
  *------------------------------------------
  */
-static int decode_zip(Bytef* dest, uLongf* destLen, const Bytef* source, uLong sourceLen)
+int decode_zip(char *dest, unsigned long* destLen, const char* source, unsigned long sourceLen)
 {
 	z_stream stream;
 	int err;
@@ -290,6 +314,37 @@ static int decode_zip(Bytef* dest, uLongf* destLen, const Bytef* source, uLong s
 	err = inflateEnd(&stream);
 	return err;
 }
+
+int encode_zip(char *dest, unsigned long* destLen, const char* source, unsigned long sourceLen) {
+	z_stream stream;
+	int err;
+
+	stream.next_in = (Bytef*)source;
+	stream.avail_in = (uInt)sourceLen;
+	/* Check for source > 64K on 16-bit machine: */
+	if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR;
+
+	stream.next_out = dest;
+	stream.avail_out = (uInt)*destLen;
+	if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR;
+
+	stream.zalloc = (alloc_func)0;
+	stream.zfree = (free_func)0;
+
+	err = zlib_deflateInit(&stream,Z_DEFAULT_COMPRESSION);
+	if (err != Z_OK) return err;
+
+	err = zlib_deflate(&stream, Z_FINISH);
+	if (err != Z_STREAM_END) {
+		zlib_inflateEnd(&stream);
+		return err == Z_OK ? Z_BUF_ERROR : err;
+	}
+	*destLen = stream.total_out;
+
+	err = zlib_deflateEnd(&stream);
+	return err;
+}
+
 /***********************************************************
  ***                File List Sobroutines                ***
  ***********************************************************/
@@ -924,6 +979,22 @@ void grfio_init(char *fname)
 	char line[1024], w1[1024], w2[1024];
 	int result = 0, result2 = 0, result3 = 0, result4 = 0;
 
+#ifdef _WIN32
+	if(!zlib_dll) {
+		zlib_dll = LoadLibrary("zlib.dll");
+		(FARPROC)zlib_inflateInit_ = GetProcAddress(zlib_dll,"inflateInit_");
+		(FARPROC)zlib_inflate      = GetProcAddress(zlib_dll,"inflate");
+		(FARPROC)zlib_inflateEnd   = GetProcAddress(zlib_dll,"inflateEnd");
+		(FARPROC)zlib_deflateInit_ = GetProcAddress(zlib_dll,"deflateInit_");
+		(FARPROC)zlib_deflate      = GetProcAddress(zlib_dll,"deflate");
+		(FARPROC)zlib_deflateEnd   = GetProcAddress(zlib_dll,"deflateEnd");
+		if(zlib_dll == NULL) {
+			MessageBox(NULL,"Can't load zlib.dll","grfio.c",MB_OK);
+			exit(1);
+		}
+	}
+#endif
+
 	data_conf = fopen(fname, "r");
 
 	// It will read, if there is grf-files.txt.

src/common/grfio.h

@@ -8,6 +8,9 @@ void* grfio_read(char*);		// GRFIO data file read
 void* grfio_reads(char*,int*);	// GRFIO data file read & size get
 int grfio_size(char*);			// GRFIO data file size get
 
+int decode_zip(char *dest, unsigned long* destLen, const char* source, unsigned long sourceLen);
+int encode_zip(char *dest, unsigned long* destLen, const char* source, unsigned long sourceLen);
+
 // Accessor to GRF filenames
 char *grfio_setdatafile(const char *str);
 char *grfio_setadatafile(const char *str);

src/map/map.c

@@ -150,20 +150,9 @@ struct charid2nick {
 	int req_id;
 };
 
-//各マップごとの最小限情報を入れるもの、READ_FROM_BITMAP用
-/*typedef struct{
-	char fn[32];//ファイル名
-	int xs,ys; //幅と高さ
-	int sizeinint;//intでの大きさ、1intに32セルの情報が入てる
-	int celltype[MAX_CELL_TYPE];//マップごとにそのタイプのセルがあれば対応する数字が入る、なければ1
-						//(タイプ1そのものは0と同じ配列gat_fileused[0]に
-	long pos[MAX_CELL_TYPE];//ビットマップファイルでの場所、読み出す時に使う
-} CELL_INFO;*/
-
-//#define READ_FROM_GAT 0 //gatファイルから
-//#define READ_FROM_BITMAP 1 //ビットマップファイルから
-int  map_read_flag = READ_FROM_GAT;//上の判定フラグ,どっちを使うかはmap_athana.conf内のread_map_from_bitmapで指定
-					//0ならばREAD_FROM_GAT,1ならばREAD_FROM_BITMAP
+int  map_read_flag = READ_FROM_GAT;
+// マップキャッシュ利用フラグ,どっちを使うかはmap_athana.conf内のread_map_from_bitmapで指定
+// 0ならば利用しない、1だと非圧縮保存、2だと圧縮して保存
 int map_getcell(int,int x,int y,CELL_CHK cellchk);
 int map_getcellp(struct map_data* m,int x,int y,CELL_CHK cellchk);
 
@@ -1569,9 +1558,9 @@ static int map_cache_open(char *fn) {
 			return 1;
 		}
 		fclose(map_cache.fp);
-	} else {
-		map_read_flag = CREATE_BITMAP;		
-	}		
+	} else if (map_read_flag == READ_FROM_BITMAP || map_read_flag == READ_FROM_BITMAP_COMPRESSED)
+		++map_read_flag;	// set to CREATE flag
+
 	// 読み込みに失敗したので新規に作成する
 	map_cache.fp = fopen(fn,"wb");
 	if(map_cache.fp) {
@@ -1611,10 +1600,8 @@ int map_cache_read(struct map_data *m) {
 			if(map_cache.map[i].water_height != map_waterheight(m->name)) {
 				// 水場の高さが違うので読み直し
 				return 0;
-			} else if(map_cache.map[i].compressed) {
-				// 圧縮ファイルは未対応
-				return 0;
-			} else {
+			} else if(map_cache.map[i].compressed == 0) {
+				// 非圧縮ファイル
 				int size = map_cache.map[i].xs * map_cache.map[i].ys;
 				m->xs = map_cache.map[i].xs;
 				m->ys = map_cache.map[i].ys;
@@ -1625,12 +1612,36 @@ int map_cache_read(struct map_data *m) {
 					return 1;
 				} else {
 					// なぜかファイル後半が欠けてるので読み直し
-					m->xs = 0;
-					m->ys = 0;
-					free(m->gat);
-					m->gat = NULL;
+					m->xs = 0; m->ys = 0; m->gat = NULL; free(m->gat);
+					return 0;
+				}
+			} else if(map_cache.map[i].compressed == 1) {
+				// 圧縮フラグ=1 : zlib
+				unsigned char *buf;
+				unsigned long dest_len;
+				int size_compress = map_cache.map[i].compressed_len;
+				m->xs = map_cache.map[i].xs;
+				m->ys = map_cache.map[i].ys;
+				m->gat = (unsigned char *)aMalloc(m->xs * m->ys * sizeof(unsigned char));
+				buf = (unsigned char*)aMalloc(size_compress);
+				fseek(map_cache.fp,map_cache.map[i].pos,SEEK_SET);
+				if(fread(buf,1,size_compress,map_cache.fp) != size_compress) {
+					// なぜかファイル後半が欠けてるので読み直し
+					printf("fread error\n");
+					m->xs = 0; m->ys = 0; m->gat = NULL;
+					free(m->gat); free(buf);
 					return 0;
 				}
+				dest_len = m->xs * m->ys;
+				decode_zip(m->gat,&dest_len,buf,size_compress);
+				if(dest_len != map_cache.map[i].xs * map_cache.map[i].ys) {
+					// 正常に解凍が出来てない
+					m->xs = 0; m->ys = 0; m->gat = NULL;
+					free(m->gat); free(buf);
+					return 0;
+				}
+				free(buf);
+				return 1;
 			}
 		}
 	}
@@ -1639,29 +1650,52 @@ int map_cache_read(struct map_data *m) {
 
 static int map_cache_write(struct map_data *m) {
 	int i;
+	unsigned long len_new , len_old;
+	char *write_buf;
 	if(!map_cache.fp) { return 0; }
 	for(i = 0;i < map_cache.head.nmaps ; i++) {
 		if(!strcmp(m->name,map_cache.map[i].fn)) {
 			// 同じエントリーがあれば上書き
-			if(
-				map_cache.map[i].xs == m->xs && map_cache.map[i].ys == m->ys &&
-				!map_cache.map[i].compressed
-			) {
-				// 幅と高さ同じで圧縮してないなら場所は変わらない
+			if(map_cache.map[i].compressed == 0) {
+				len_old = map_cache.map[i].xs * map_cache.map[i].ys;
+			} else if(map_cache.map[i].compressed == 1) {
+				len_old = map_cache.map[i].compressed_len;
+			} else {
+				// サポートされてない形式なので長さ０
+				len_old = 0;
+			}
+			if(map_read_flag >= READ_FROM_BITMAP_COMPRESSED) {
+				// 圧縮保存
+				// さすがに２倍に膨れる事はないという事で
+				write_buf = aMalloc(m->xs * m->ys * 2);
+				len_new = m->xs * m->ys * 2;
+				encode_zip(write_buf,&len_new,m->gat,m->xs * m->ys);
+				map_cache.map[i].compressed     = 1;
+				map_cache.map[i].compressed_len = len_new;
+			} else {
+				len_new = m->xs * m->ys;
+				write_buf = m->gat;
+				map_cache.map[i].compressed     = 0;
+				map_cache.map[i].compressed_len = 0;	
+			}
+			if(len_new <= len_old) {
+				// サイズが同じか小さくなったので場所は変わらない
 				fseek(map_cache.fp,map_cache.map[i].pos,SEEK_SET);
-				fwrite(m->gat,m->xs,m->ys,map_cache.fp);
+				fwrite(write_buf,1,len_new,map_cache.fp);
 			} else {
-				// 幅と高さが違うなら新しい場所に登録
-				int size = m->xs * m->ys;
+				// 新しい場所に登録
 				fseek(map_cache.fp,map_cache.head.filesize,SEEK_SET);
-				fwrite(m->gat,1,size,map_cache.fp);
+				fwrite(write_buf,1,len_new,map_cache.fp);
 				map_cache.map[i].pos = map_cache.head.filesize;
-				map_cache.map[i].xs  = m->xs;
-				map_cache.map[i].ys  = m->ys;
-				map_cache.head.filesize += size;
+				map_cache.head.filesize += len_new;
 			}
+			map_cache.map[i].xs  = m->xs;
+			map_cache.map[i].ys  = m->ys;
 			map_cache.map[i].water_height = map_waterheight(m->name);
 			map_cache.dirty = 1;
+			if(map_read_flag >= READ_FROM_BITMAP_COMPRESSED) {
+				free(write_buf);
+			}
 			return 0;
 		}
 	}
@@ -1669,16 +1703,30 @@ static int map_cache_write(struct map_data *m) {
 	for(i = 0;i < map_cache.head.nmaps ; i++) {
 		if(map_cache.map[i].fn[0] == 0) {
 			// 新しい場所に登録
-			int size = m->xs * m->ys;
+			if(map_read_flag >= READ_FROM_BITMAP_COMPRESSED) {
+				write_buf = aMalloc(m->xs * m->ys * 2);
+				len_new = m->xs * m->ys * 2;
+				encode_zip(write_buf,&len_new,m->gat,m->xs * m->ys);
+				map_cache.map[i].compressed     = 1;
+				map_cache.map[i].compressed_len = len_new;
+			} else {
+				len_new = m->xs * m->ys;
+				write_buf = m->gat;
+				map_cache.map[i].compressed     = 0;
+				map_cache.map[i].compressed_len = 0;
+			}
 			strncpy(map_cache.map[i].fn,m->name,sizeof(map_cache.map[0].fn));
 			fseek(map_cache.fp,map_cache.head.filesize,SEEK_SET);
-			fwrite(m->gat,1,size,map_cache.fp);
+			fwrite(write_buf,1,len_new,map_cache.fp);
 			map_cache.map[i].pos = map_cache.head.filesize;
 			map_cache.map[i].xs  = m->xs;
 			map_cache.map[i].ys  = m->ys;
 			map_cache.map[i].water_height = map_waterheight(m->name);
-			map_cache.head.filesize += size;
+			map_cache.head.filesize += len_new;
 			map_cache.dirty = 1;
+			if(map_read_flag >= READ_FROM_BITMAP_COMPRESSED) {
+				free(write_buf);
+			}
 			return 0;
 		}
 	}
@@ -1931,13 +1979,14 @@ int map_readallmap(void) {
 	int map_cache = 0;
 	
 	// マップキャッシュを開く
-	if(map_read_flag == READ_FROM_BITMAP) {
+	if(map_read_flag >= READ_FROM_BITMAP) {
 		map_cache_open(map_bitmap_filename);
 	}
 
 	sprintf(tmp_output, "Loading Maps%s...\n",
-		(map_read_flag == CREATE_BITMAP ? " (Generating Map Cache)" :
-		map_read_flag == READ_FROM_BITMAP ? " (w/ Map Cache)" :
+		(map_read_flag == CREATE_BITMAP_COMPRESSED ? " (Generating Map Cache w/ Compression)" :
+		map_read_flag == CREATE_BITMAP ? " (Generating Map Cache)" :
+		map_read_flag >= READ_FROM_BITMAP ? " (w/ Map Cache)" :
 		map_read_flag == READ_FROM_AFM ? " (w/ AFM)" : ""));
 	ShowStatus(tmp_output);
 
@@ -2001,8 +2050,8 @@ int map_readallmap(void) {
 	ShowInfo(tmp_output);
 	
 	map_cache_close();
-	if(map_read_flag == CREATE_BITMAP) {
-		map_read_flag = READ_FROM_BITMAP;
+	if(map_read_flag == CREATE_BITMAP || map_read_flag == CREATE_BITMAP_COMPRESSED) {
+		--map_read_flag;
 	}
 	
 	if (maps_removed) {
@@ -2214,10 +2263,14 @@ int map_config_read(char *cfgName) {
 			} else if (strcmpi(w1, "mapreg_txt") == 0) {
 				strcpy(mapreg_txt, w2);
 			}else if(strcmpi(w1,"read_map_from_bitmap")==0){
-				if (atoi(w2) == 1)
+				if (atoi(w2) == 2)
+					map_read_flag = READ_FROM_BITMAP_COMPRESSED;
+				else if (atoi(w2) == 1)
 					map_read_flag = READ_FROM_BITMAP;
 				else
 					map_read_flag = READ_FROM_GAT;
+			}else if(strcmpi(w1,"map_bitmap_path")==0){
+				strncpy(map_bitmap_filename,w2,255);
 			} else if (strcmpi(w1, "import") == 0) {
 				map_config_read(w2);
 			} else if (strcmpi(w1, "console") == 0) {
@@ -2408,7 +2461,9 @@ int sql_config_read(char *cfgName)
 		} else if(strcmpi(w1,"log_db_port")==0) {
 			log_db_port = atoi(w2);
 		}else if(strcmpi(w1,"read_map_from_bitmap")==0){
-			if (atoi(w2) == 1)
+			if (atoi(w2) == 2)
+				map_read_flag = READ_FROM_BITMAP_COMPRESSED;
+			else if (atoi(w2) == 1)
 				map_read_flag = READ_FROM_BITMAP;
 			else
 				map_read_flag = READ_FROM_GAT;

src/map/map.h

@@ -668,7 +668,11 @@ extern int night_flag; // 0=day, 1=night [Yor]
 int map_getcell(int,int,int,CELL_CHK);
 int map_getcellp(struct map_data*,int,int,CELL_CHK);
 extern int map_read_flag;	//セル情報のソース判定フラグ、0ならgrfファイル、1ならビットマップファイル
-enum { READ_FROM_GAT, READ_FROM_AFM, READ_FROM_BITMAP, CREATE_BITMAP };
+enum { 
+	READ_FROM_GAT, READ_FROM_AFM,
+	READ_FROM_BITMAP, CREATE_BITMAP,
+	READ_FROM_BITMAP_COMPRESSED, CREATE_BITMAP_COMPRESSED
+};
 
 extern char motd_txt[];
 extern char help_txt[];