FileSystem: Change file lookup to match all extensions

Previously, file lookup (ie. extension supplementing) would only
try out a few predetermined extensions based on the asset type.
This was not accurate in regard to RMXP's behavior, which will
happily match "some_asset" against "some_asset.abcef" and try
to open it.

Some games make use of this quirk and rename their ogg audio files
to "*.dat" or similar to thwart users from copying them.

This change also makes it easier to read arbitrary formats
supported by SDL_image without modifying mkxp.

src/alstream.cpp

@@ -196,8 +196,8 @@ void ALStream::closeSource()
 
 void ALStream::openSource(const std::string &filename)
 {
-	const char *ext;
+	char ext[8];
-	shState->fileSystem().openRead(srcOps, filename.c_str(), FileSystem::Audio, false, &ext);
+	shState->fileSystem().openRead(srcOps, filename.c_str(), false, ext, sizeof(ext));
 	needsRewind.clear();
 
 	/* Try to read ogg file signature */

src/bitmap.cpp

@@ -236,9 +236,10 @@ struct BitmapPrivate
 Bitmap::Bitmap(const char *filename)
 {
 	SDL_RWops ops;
-	const char *extension;
+	char ext[8];
-	shState->fileSystem().openRead(ops, filename, FileSystem::Image, false, &extension);
+
-	SDL_Surface *imgSurf = IMG_LoadTyped_RW(&ops, 1, extension);
+	shState->fileSystem().openRead(ops, filename, false, ext, sizeof(ext));
+	SDL_Surface *imgSurf = IMG_LoadTyped_RW(&ops, 1, ext);
 
 	if (!imgSurf)
 		throw Exception(Exception::SDLError, "Error loading image '%s': %s",

src/filesystem.cpp

@@ -230,18 +230,36 @@ static int SDL_RWopsCloseFree(SDL_RWops *ops)
 	return result;
 }
 
+/* Copies the first srcN characters from src into dst,
+ * or the full string if srcN == -1. Never writes more
+ * than dstMax, and guarantees dst to be null terminated.
+ * Returns copied bytes (minus terminating null) */
+static size_t
+strcpySafe(char *dst, const char *src,
+           size_t dstMax, int srcN)
+{
+	if (srcN < 0)
+		srcN = strlen(src);
+
+	size_t cpyMax = std::min<size_t>(dstMax-1, srcN);
+
+	memcpy(dst, src, cpyMax);
+	dst[cpyMax] = '\0';
+
+	return cpyMax;
+}
+
 const Uint32 SDL_RWOPS_PHYSFS = SDL_RWOPS_UNKNOWN+10;
 
 struct FileSystemPrivate
 {
-	/* Maps: lower case filename, To: actual (mixed case) filename.
+	/* Maps: lower case filepath without extension,
+	 * To:   mixed case full filepath
 	 * This is for compatibility with games that take Windows'
 	 * case insensitivity for granted */
 	BoostHash<std::string, std::string> pathCache;
 	bool havePathCache;
 
-	std::vector<std::string> extensions[FileSystem::Undefined+1];
-
 	/* Attempt to locate an extension string in a filename.
 	 * Either a pointer into the input string pointing at the
 	 * extension, or null is returned */
@@ -261,120 +279,129 @@ struct FileSystemPrivate
 		return 0;
 	}
 
-	/* Complete filename via regular physfs lookup */
+	struct CompleteFilenameData
-	bool completeFilenameReg(const char *filename,
+	{
-	                         FileSystem::FileType type,
+		bool found;
+		/* Contains the incomplete filename we're looking for;
+		 * when found, we write the complete filename into this
+		 * same buffer */
+		char *outBuf;
+		/* Length of incomplete file name */
+		size_t filenameLen;
+		/* Maximum we can write into outBuf */
+		size_t outBufN;
+	};
+
+	static void completeFilenameRegCB(void *data, const char *,
+	                                  const char *fname)
+	{
+		CompleteFilenameData &d = *static_cast<CompleteFilenameData*>(data);
+
+		if (d.found)
+			return;
+
+		if (strncmp(d.outBuf, fname, d.filenameLen) != 0)
+			return;
+
+		/* If fname matches up to a following '.' (meaning the rest is part
+		 * of the extension), or up to a following '\0' (full match), we've
+		 * found our file */
+		switch (fname[d.filenameLen])
+		{
+		case '.' :
+			/* Overwrite the incomplete file name we looked for with
+			 * the full version containing any extensions */
+			strcpySafe(d.outBuf, fname, d.outBufN, -1);
+		case '\0' :
+			d.found = true;
+		}
+	}
+
+	bool completeFilenameReg(const char *filepath,
 	                         char *outBuffer,
-	                         size_t outN,
+	                         size_t outN)
-	                         const char **foundExt)
 	{
-		/* Try supplementing extensions to find an existing path */
+		strcpySafe(outBuffer, filepath, outN, -1);
-		const std::vector<std::string> &extList = extensions[type];
 
-		for (size_t i = 0; i < extList.size(); ++i)
+		size_t len = strlen(outBuffer);
+		char *delim;
+
+		/* Find the deliminator separating directory and file name */
+		for (delim = outBuffer + len; delim > outBuffer; --delim)
+			if (*delim == '/')
+				break;
+
+		bool root = (delim == outBuffer);
+		CompleteFilenameData d;
+
+		if (!root)
 		{
-			const char *ext = extList[i].c_str();
+			/* If we have such a deliminator, we set it to '\0' so we
+			 * can pass the first half to PhysFS as the directory name,
+			 * and compare all filenames against the second half */
+			d.outBuf = delim+1;
+			d.filenameLen = len - (delim - outBuffer + 1);
 
-			snprintf(outBuffer, outN, "%s.%s", filename, ext);
+			*delim = '\0';
-
+		}
-			if (PHYSFS_exists(outBuffer))
+		else
-			{
+		{
-				if (foundExt)
+			/* Otherwise the file is in the root directory */
-					*foundExt = ext;
+			d.outBuf = outBuffer;
-
+			d.filenameLen = len - (delim - outBuffer);
-				return true;
-			}
 		}
 
-		/* Doing the check without supplemented extension
+		d.found = false;
-		 * fits the usage pattern of RMXP games */
+		d.outBufN = outN - (d.outBuf - outBuffer);
-		if (PHYSFS_exists(filename))
-		{
-			strncpy(outBuffer, filename, outN);
 
-			if (foundExt)
+		PHYSFS_enumerateFilesCallback(root ? "" : outBuffer, completeFilenameRegCB, &d);
-				*foundExt = findExt(filename);
 
-			return true;
+		if (!d.found)
-		}
+			return false;
 
-		return false;
+		/* Now we put the deliminator back in to form the completed
+		 * file path (if required) */
+		if (delim != outBuffer)
+			*delim = '/';
+
+		return true;
 	}
 
-	/* Complete filename via path cache */
+	bool completeFilenamePC(const char *filepath,
-	bool completeFilenamePC(const char *filename,
-	                        FileSystem::FileType type,
 	                        char *outBuffer,
-	                        size_t outN,
+	                        size_t outN)
-	                        const char **foundExt)
 	{
-		size_t i;
+		std::string lowCase(filepath);
-		char lowCase[512];
 
-		for (i = 0; i < sizeof(lowCase)-1 && filename[i]; ++i)
+		for (size_t i = 0; i < lowCase.size(); ++i)
-			lowCase[i] = tolower(filename[i]);
+			lowCase[i] = tolower(lowCase[i]);
 
-		lowCase[i] = '\0';
+		if (!pathCache.contains(lowCase))
+			return false;
 
-		std::string key;
+		const std::string &fullPath = pathCache[lowCase];
+		strcpySafe(outBuffer, fullPath.c_str(), outN, fullPath.size());
 
-		const std::vector<std::string> &extList = extensions[type];
+		return true;
-
-		for (size_t i = 0; i < extList.size(); ++i)
-		{
-			const char *ext = extList[i].c_str();
-
-			snprintf(outBuffer, outN, "%s.%s", lowCase, ext);
-			key = outBuffer;
-
-			if (pathCache.contains(key))
-			{
-				strncpy(outBuffer, pathCache[key].c_str(), outN);
-
-				if (foundExt)
-					*foundExt = ext;
-
-				return true;
-			}
-		}
-
-		key = lowCase;
-
-		if (pathCache.contains(key))
-		{
-			strncpy(outBuffer, pathCache[key].c_str(), outN);
-
-			if (foundExt)
-				*foundExt = findExt(filename);
-
-			return true;
-		}
-
-		return false;
 	}
 
-	/* Try to complete 'filename' with file extensions
+	bool completeFilename(const char *filepath,
-	 * based on 'type'. If no combination could be found,
-	 * returns false, and 'foundExt' is untouched */
-	bool completeFileName(const char *filename,
-	                      FileSystem::FileType type,
 	                      char *outBuffer,
-	                      size_t outN,
+	                      size_t outN)
-	                      const char **foundExt)
 	{
 		if (havePathCache)
-			return completeFilenamePC(filename, type, outBuffer, outN, foundExt);
+			return completeFilenamePC(filepath, outBuffer, outN);
 		else
-			return completeFilenameReg(filename, type, outBuffer, outN, foundExt);
+			return completeFilenameReg(filepath, outBuffer, outN);
 	}
 
 	PHYSFS_File *openReadHandle(const char *filename,
-	                            FileSystem::FileType type,
+	                            char *extBuf,
-	                            const char **foundExt)
+	                            size_t extBufN)
 	{
 		char found[512];
 
-		if (!completeFileName(filename, type, found, sizeof(found), foundExt))
+		if (!completeFilename(filename, found, sizeof(found)))
 			throw Exception(Exception::NoFileError, "%s", filename);
 
 		PHYSFS_File *handle = PHYSFS_openRead(found);
@@ -382,6 +409,21 @@ struct FileSystemPrivate
 		if (!handle)
 			throw Exception(Exception::PHYSFSError, "PhysFS: %s", PHYSFS_getLastError());
 
+		if (!extBuf)
+			return handle;
+
+		for (char *q = found+strlen(found); q > found; --q)
+		{
+			if (*q == '/')
+				break;
+
+			if (*q != '.')
+				continue;
+
+			strcpySafe(extBuf, q+1, extBufN, -1);
+			break;
+		}
+
 		return handle;
 	}
 
@@ -408,45 +450,8 @@ FileSystem::FileSystem(const char *argv0,
                        bool allowSymlinks)
 {
 	p = new FileSystemPrivate;
-
 	p->havePathCache = false;
 
-	/* Image extensions */
-	p->extensions[Image].push_back("jpg");
-	p->extensions[Image].push_back("png");
-
-	/* Audio extensions */
-	const Sound_DecoderInfo **di;
-	for (di = Sound_AvailableDecoders(); *di; ++di)
-	{
-		const char **ext;
-		for (ext = (*di)->extensions; *ext; ++ext)
-		{
-			/* All reported extensions are uppercase,
-			 * so we need to hammer them down first */
-			char buf[16];
-			for (size_t i = 0; i < sizeof(buf); ++i)
-			{
-				buf[i] = tolower((*ext)[i]);
-
-				if (!buf[i])
-					break;
-			}
-
-			p->extensions[Audio].push_back(buf);
-		}
-	}
-
-	if (rgssVer >= 2 && !contains(p->extensions[Audio], std::string("ogg")))
-		p->extensions[Audio].push_back("ogg");
-
-	p->extensions[Audio].push_back("mid");
-	p->extensions[Audio].push_back("midi");
-
-	/* Font extensions */
-	p->extensions[Font].push_back("ttf");
-	p->extensions[Font].push_back("otf");
-
 	PHYSFS_init(argv0);
 
 	PHYSFS_registerArchiver(&RGSS1_Archiver);
@@ -544,12 +549,22 @@ static void cacheEnumCB(void *d, const char *origdir,
 
 	std::string mixedCase(ptr);
 
-	for (char *p = bufNfc; *p; ++p)
+	for (char *q = bufNfc; *q; ++q)
-		*p = tolower(*p);
+		*q = tolower(*q);
 
-	std::string lowerCase(ptr);
+	p->pathCache.insert(std::string(ptr), mixedCase);
 
-	p->pathCache.insert(lowerCase, mixedCase);
+	for (char *q = ptr+strlen(ptr); q > ptr; --q)
+	{
+		if (*q == '/')
+			break;
+
+		if (*q != '.')
+			continue;
+
+		*q = '\0';
+		p->pathCache.insert(std::string(ptr), mixedCase);
+	}
 
 	PHYSFS_enumerateFilesCallback(mixedCase.c_str(), cacheEnumCB, d);
 }
@@ -558,7 +573,7 @@ void FileSystem::createPathCache()
 {
 #ifdef __APPLE__
 	CacheEnumCBData data(p);
-	PHYSFS_enumerateFilesCallback("", cacheEnumCB, &data);
+	PHYSFS_enumerateFilesCallback("", cacheEnumCB2, &data);
 #else
 	PHYSFS_enumerateFilesCallback("", cacheEnumCB, p);
 #endif
@@ -566,12 +581,6 @@ void FileSystem::createPathCache()
 	p->havePathCache = true;
 }
 
-static void strToLower(std::string &str)
-{
-	for (size_t i = 0; i < str.size(); ++i)
-		str[i] = tolower(str[i]);
-}
-
 struct FontSetsCBData
 {
 	FileSystemPrivate *p;
@@ -590,16 +599,21 @@ static void fontSetEnumCB(void *data, const char *,
 	if (!ext)
 		return;
 
-	std::string lower(ext);
+	char lowExt[8];
-	strToLower(lower);
+	size_t i;
 
-	if (!contains(p->extensions[FileSystem::Font], lower))
+	for (i = 0; i < sizeof(lowExt)-1 && ext[i]; ++i)
+		lowExt[i] = tolower(ext[i]);
+	lowExt[i] = '\0';
+
+	if (strcmp(lowExt, "ttf") && strcmp(lowExt, "otf"))
 		return;
 
-	std::string filename("Fonts/");
+	char filename[512];
-	filename += fname;
+	snprintf(filename, sizeof(filename), "Fonts/%s", fname);
+	filename[sizeof(filename)-1] = '\0';
 
-	PHYSFS_File *handle = PHYSFS_openRead(filename.c_str());
+	PHYSFS_File *handle = PHYSFS_openRead(filename);
 
 	if (!handle)
 		return;
@@ -621,11 +635,11 @@ void FileSystem::initFontSets(SharedFontState &sfs)
 
 void FileSystem::openRead(SDL_RWops &ops,
                           const char *filename,
-                          FileType type,
                           bool freeOnClose,
-                          const char **foundExt)
+                          char *extBuf,
+                          size_t extBufN)
 {
-	PHYSFS_File *handle = p->openReadHandle(filename, type, foundExt);
+ 	PHYSFS_File *handle = p->openReadHandle(filename, extBuf, extBufN);
 
 	p->initReadOps(handle, ops, freeOnClose);
 }
@@ -640,9 +654,9 @@ void FileSystem::openReadRaw(SDL_RWops &ops,
 	p->initReadOps(handle, ops, freeOnClose);
 }
 
-bool FileSystem::exists(const char *filename, FileType type)
+bool FileSystem::exists(const char *filename)
 {
 	char found[512];
 
-	return p->completeFileName(filename, type, found, sizeof(found), 0);
+	return p->completeFilename(filename, found, sizeof(found));
 }

src/filesystem.h

@@ -43,28 +43,18 @@ public:
 	 * available font assets */
 	void initFontSets(SharedFontState &sfs);
 
-	/* For extension supplementing */
-	enum FileType
-	{
-		Image = 0,
-		Audio,
-		Font,
-		Undefined
-	};
-
 	void openRead(SDL_RWops &ops,
 	              const char *filename,
-	              FileType type = Undefined,
 	              bool freeOnClose = false,
-	              const char **foundExt = 0);
+	              char *extBuf = 0,
+	              size_t extBufN = 0);
 
 	/* Circumvents extension supplementing */
 	void openReadRaw(SDL_RWops &ops,
 	                 const char *filename,
 	                 bool freeOnClose = false);
 
-	bool exists(const char *filename,
+	bool exists(const char *filename);
-	            FileType type = Undefined);
 
 private:
 	FileSystemPrivate *p;

src/soundemitter.cpp

@@ -197,12 +197,12 @@ SoundBuffer *SoundEmitter::allocateBuffer(const std::string &filename)
 	{
 		/* Buffer not in cashe, needs to be loaded */
 		SDL_RWops dataSource;
-		const char *extension;
+		char ext[8];
 
 		shState->fileSystem().openRead(dataSource, filename.c_str(),
-									   FileSystem::Audio, false, &extension);
+		                               false, ext, sizeof(ext));
 
-		Sound_Sample *sampleHandle = Sound_NewSample(&dataSource, extension, 0, STREAM_BUF_SIZE);
+		Sound_Sample *sampleHandle = Sound_NewSample(&dataSource, ext, 0, STREAM_BUF_SIZE);
 
 		if (!sampleHandle)
 		{