Tilemap: Use vertex shader based autotile animation strategy

Previously, we would just stuff the entire tilemap vertex data
four times into the buffers, with only the autotile vertices
offset according to the animation frame. This meant we could
prepare the buffers once, and then just bind a different offset
for each animation frame without any shader changes, but it also
lead to a huge amount of data being duplicated (and blowing up
the buffer sizes).

The new method only requires one buffer, and instead animates by
recognizing vertices belonging to autotiles in a custom vertex
shader, which offsets them on the fly according to the animation
index.

With giant tilemaps, this method would turn out to be a little
less efficient, but considering the Tilemap is planned to be
rewritten to only hold the range of tiles visible on the screen
in its buffers, the on the fly offsetting will become neglient,
while at the same time the amount of data we have to send to the
GPU everytime the tilemap is updated is greatly reduced; so a
net win in the end.

mkxp.pro

@@ -168,6 +168,7 @@ EMBED = \
 	shader/simple.vert \
 	shader/simpleColor.vert \
 	shader/sprite.vert \
+	shader/tilemap.vert \
 	assets/liberation.ttf
 
 RGSS2 {

shader/tilemap.vert

@@ -0,0 +1,27 @@
+
+uniform mat4 projMat;
+
+uniform vec2 texSizeInv;
+uniform vec2 translation;
+
+uniform float aniIndex;
+
+attribute vec2 position;
+attribute vec2 texCoord;
+
+varying vec2 v_texCoord;
+
+const float atAreaW = 96.0;
+const float atAreaH = 128.0*7.0;
+const float atAniOffset = 32.0*3.0;
+
+void main()
+{
+	vec2 tex = texCoord;
+	if (tex.x <= atAreaW && tex.y <= atAreaH)
+		tex.x += aniIndex * atAniOffset;
+
+	gl_Position = projMat * vec4(position + translation, 0, 1);
+
+	v_texCoord = tex * texSizeInv;
+}

src/shader.cpp

@@ -40,6 +40,7 @@
 #include "../simple.vert.xxd"
 #include "../simpleColor.vert.xxd"
 #include "../sprite.vert.xxd"
+#include "../tilemap.vert.xxd"
 
 #ifdef RGSS2
 #include "../blur.frag.xxd"
@@ -422,6 +423,22 @@ void PlaneShader::setOpacity(float value)
 }
 
 
+TilemapShader::TilemapShader()
+{
+	INIT_SHADER(tilemap, simple, TilemapShader);
+
+	ShaderBase::init();
+
+	GET_U(aniIndex);
+}
+
+void TilemapShader::setAniIndex(int value)
+{
+	gl.Uniform1f(u_aniIndex, value);
+}
+
+
+
 FlashMapShader::FlashMapShader()
 {
 	INIT_SHADER(simpleColor, flashMap, FlashMapShader);

src/shader.h

@@ -179,6 +179,17 @@ private:
 	GLint u_tone, u_color, u_flash, u_opacity;
 };
 
+class TilemapShader : public ShaderBase
+{
+public:
+	TilemapShader();
+
+	void setAniIndex(int value);
+
+private:
+	GLint u_aniIndex;
+};
+
 class FlashMapShader : public ShaderBase
 {
 public:
@@ -259,6 +270,7 @@ struct ShaderSet
 	SimpleSpriteShader simpleSprite;
 	SpriteShader sprite;
 	PlaneShader plane;
+	TilemapShader tilemap;
 	FlashMapShader flashMap;
 	TransShader trans;
 	SimpleTransShader simpleTrans;

src/tileatlas.cpp

@@ -47,6 +47,8 @@ static const int atArea = atAreaW * atAreaH;
 static const int tilesetW = 256;
 static const int tsLaneW = tilesetW / 2;
 
+static const int atBuffer = 32;
+
 static int freeArea(int width, int height)
 {
 	return width * height - atArea;
@@ -55,7 +57,7 @@ static int freeArea(int width, int height)
 Vec2i minSize(int tilesetH, int maxAtlasSize)
 {
 	int width = atAreaW;
-	int height = atAreaH;
+	int height = atAreaH + atBuffer;
 
 	const int tsArea = tilesetW * tilesetH;
 
@@ -93,7 +95,7 @@ static ColumnVec calcDstCols(int atlasW, int atlasH)
 	cols.reserve(3);
 
 	/* Columns below the autotile area */
-	const int underAt = atlasH - atAreaH;
+	const int underAt = atlasH - (atAreaH + atBuffer);
 
 	for (int i = 0; i < 3; ++i)
 		cols.push_back(Column(i*tsLaneW, atAreaH, underAt));
@@ -174,7 +176,7 @@ Vec2i tileToAtlasCoor(int tileX, int tileY, int tilesetH, int atlasH)
 	int laneY = tileY*32;
 
 	int longlaneH = atlasH;
-	int shortlaneH = longlaneH - atAreaH;
+	int shortlaneH = longlaneH - (atAreaH + atBuffer);
 
 	int longlaneOffset = shortlaneH * 3;
 

src/tilemap.cpp

@@ -46,7 +46,6 @@
 extern const StaticRect autotileRects[];
 
 typedef std::vector<SVertex> SVVector;
-typedef struct { SVVector v[4]; } TileVBuffer;
 
 static const int tilesetW  = 8 * 32;
 static const int autotileW = 3 * 32;
@@ -75,6 +74,9 @@ static const int tsLaneW = tilesetW / 2;
  *   other half (as if the right half was cut and pasted below
  *   the left half before fitting it all into the atlas).
  *   Internally these halves are called "tileset lanes".
+ *   There is a 32 pixel wide empty buffer below the autotile
+ *   area so the vertex shader can safely differentiate between
+ *   autotile and tileset vertices (relevant for autotile animation).
  *
  *                  Tile atlas
  *   *-----------------------*--------------*
@@ -92,6 +94,8 @@ static const int tsLaneW = tilesetW / 2;
  *   | AT7 |     |     |     |   v   ¦   v   |
  *   |     |     |     |     |       ¦       |
  *   |-----|-----|-----|-----|       ¦       |
+ *   |      Empty space      |       |       |
+ *   |-----------------------|       |       |
  *   |       ¦       ¦       ¦       ¦       |
  *   | Tile- ¦   |   ¦   |   ¦       ¦       |
  *   |  set  ¦   v   ¦   v   ¦       ¦       |
@@ -107,17 +111,9 @@ static const int tsLaneW = tilesetW / 2;
  *   is used up, and then, when the max texture size
  *   is reached, horizontally.
  *
- *   To animate the autotiles, we keep 4 buffers (packed into
+ *   To animate the autotiles, we catch any autotile vertices in
- *   one big VBO and accessed using offsets) with vertex data
+ *   the tilemap shader based on their texcoord, and offset them
- *   corresponding to the respective animation frame. Likewise,
+ *   horizontally by (animation index) * (autotile frame width = 96).
- *   the IBO is expanded to 4 times its usual size. In practice
- *   this means that all vertex data which does not stem from an
- *   animated autotile is duplicated across all 4 buffers.
- *   The range of one such buffer inside the VBO is called
- *   buffer frame, and tiles.bufferFrameSize * bufferIndex gives
- *   us the base offset into the IBO to access it.
- *   If there are no animated autotiles attached, we only use
- *   the first buffer.
  *
  * Elements:
  *   Even though the Tilemap carries similarities with other
@@ -280,10 +276,10 @@ struct TilemapPrivate
 	int mapHeight;
 
 	/* Ground layer vertices */
-	TileVBuffer groundVert;
+	SVVector groundVert;
 
 	/* Scanrow vertices */
-	std::vector<TileVBuffer> scanrowVert;
+	std::vector<SVVector> scanrowVert;
 
 	/* Base quad indices of each scanrow
 	 * in the shared buffer */
@@ -460,11 +456,6 @@ struct TilemapPrivate
 		prepareCon.disconnect();
 	}
 
-	uint8_t bufferCount() const
-	{
-		return tiles.animated ? 4 : 1;
-	}
-
 	void updateAtlasInfo()
 	{
 		if (!tileset || tileset->isDisposed())
@@ -638,8 +629,19 @@ struct TilemapPrivate
 			int blitH = std::min(autotile->height(), atAreaH);
 
 			FBO::bind(autotile->getGLTypes().fbo, FBO::Read);
+
+			if (blitW <= autotileW)
+			{
+				/* Static autotile */
+				for (int j = 0; j < 4; ++j)
+					FBO::blit(0, 0, autotileW*j, atInd*autotileH, blitW, blitH);
+			}
+			else
+			{
+				/* Animated autotile */
 				FBO::blit(0, 0, 0, atInd*autotileH, blitW, blitH);
 			}
+		}
 
 		/* Blit tileset */
 		if (tileset->megaSurface())
@@ -706,7 +708,7 @@ struct TilemapPrivate
 		return FloatRect(x, y, 16, 16);
 	}
 
-	void handleAutotile(int x, int y, int tileInd, TileVBuffer *array)
+	void handleAutotile(int x, int y, int tileInd, SVVector *array)
 	{
 		/* Which autotile [0-7] */
 		int atInd = tileInd / 48 - 1;
@@ -724,20 +726,12 @@ struct TilemapPrivate
 			/* Adjust to atlas coordinates */
 			texRect.y += atInd * autotileH;
 
-			for (size_t k = 0; k < bufferCount(); ++k)
-			{
-				FloatRect _texRect = texRect;
-
-				if (contains(atlas.animatedATs, atInd))
-					_texRect.x += autotileW*k;
-
 			SVertex v[4];
-				Quad::setTexPosRect(v, _texRect, posRect);
+			Quad::setTexPosRect(v, texRect, posRect);
 
 			/* Iterate over 4 vertices */
 			for (size_t i = 0; i < 4; ++i)
-					array->v[k].push_back(v[i]);
+				array->push_back(v[i]);
-			}
 		}
 	}
 
@@ -755,7 +749,7 @@ struct TilemapPrivate
 		if (prio == -1)
 			return;
 
-		TileVBuffer *targetArray;
+		SVVector *targetArray;
 
 		/* Prio 0 tiles are all part of the same ground layer */
 		if (prio == 0)
@@ -786,15 +780,13 @@ struct TilemapPrivate
 		SVertex v[4];
 		Quad::setTexPosRect(v, texRect, posRect);
 
-		for (size_t k = 0; k < bufferCount(); ++k)
 		for (size_t i = 0; i < 4; ++i)
-				targetArray->v[k].push_back(v[i]);
+			targetArray->push_back(v[i]);
 	}
 
 	void clearQuadArrays()
 	{
-		for (size_t i = 0; i < 4; ++i)
+		groundVert.clear();
-			groundVert.v[i].clear();
 
 		scanrowVert.clear();
 		scanrowBases.clear();
@@ -830,47 +822,58 @@ struct TilemapPrivate
 		scanrowBases.resize(scanrowCount + 1);
 
 		/* Calculate total quad count */
-		size_t groundQuadCount = groundVert.v[0].size() / 4;
+		size_t groundQuadCount = groundVert.size() / 4;
 		size_t quadCount = groundQuadCount;
 
 		for (size_t i = 0; i < scanrowCount; ++i)
 		{
 			scanrowBases[i] = quadCount;
-			quadCount += scanrowVert[i].v[0].size() / 4;
+			quadCount += scanrowVert[i].size() / 4;
 		}
 
 		scanrowBases[scanrowCount] = quadCount;
 
-		size_t bufferFrameQuadCount = quadCount;
 		tiles.bufferFrameSize = quadCount * 6 * sizeof(uint32_t);
 
-		quadCount *= bufferCount();
-
 		VBO::bind(tiles.vbo);
 		VBO::allocEmpty(quadDataSize(quadCount));
 
-		for (size_t k = 0; k < bufferCount(); ++k)
+		VBO::uploadSubData(0, quadDataSize(groundQuadCount), &groundVert[0]);
-		{
-			VBO::uploadSubData(k*quadDataSize(bufferFrameQuadCount),
-			                   quadDataSize(groundQuadCount), &groundVert.v[k][0]);
 
 		for (size_t i = 0; i < scanrowCount; ++i)
 		{
-				if (scanrowVert[i].v[0].empty())
+			if (scanrowVert[i].empty())
 				continue;
 
-				VBO::uploadSubData(k*quadDataSize(bufferFrameQuadCount) + quadDataSize(scanrowBases[i]),
+			VBO::uploadSubData(quadDataSize(scanrowBases[i]),
-				                   quadDataSize(scanrowSize(i)), &scanrowVert[i].v[k][0]);
+			                   quadDataSize(scanrowSize(i)), &scanrowVert[i][0]);
-			}
 		}
 
 		VBO::unbind();
 
 		/* Ensure global IBO size */
-		shState->ensureQuadIBO(quadCount*bufferCount());
+		shState->ensureQuadIBO(quadCount);
 	}
 
-	void bindAtlas(SimpleShader &shader)
+	void bindShader(ShaderBase *&shaderVar)
+	{
+		if (tiles.animated)
+		{
+			TilemapShader &tilemapShader = shState->shaders().tilemap;
+			tilemapShader.bind();
+			tilemapShader.setAniIndex(tiles.frameIdx);
+			shaderVar = &tilemapShader;
+		}
+		else
+		{
+			shaderVar = &shState->shaders().simple;
+			shaderVar->bind();
+		}
+
+		shaderVar->applyViewportProj();
+	}
+
+	void bindAtlas(ShaderBase &shader)
 	{
 		TEX::bind(atlas.gl.tex);
 		shader.setTexSize(atlas.size);
@@ -985,7 +988,7 @@ struct TilemapPrivate
 		std::vector<int> scanrowInd;
 
 		for (size_t i = 0; i < scanrowCount; ++i)
-			if (scanrowVert[i].v[0].size() > 0)
+			if (scanrowVert[i].size() > 0)
 				scanrowInd.push_back(i);
 
 		generateElements(scanrowInd);
@@ -1110,15 +1113,14 @@ GroundLayer::GroundLayer(TilemapPrivate *p, Viewport *viewport)
 
 void GroundLayer::draw()
 {
-	SimpleShader &shader = shState->shaders().simple;
+	ShaderBase *shader;
-	shader.bind();
-	shader.applyViewportProj();
 
-	p->bindAtlas(shader);
+	p->bindShader(shader);
+	p->bindAtlas(*shader);
 
 	VAO::bind(p->tiles.vao);
 
-	p->setTranslation(Normal, shader);
+	p->setTranslation(Normal, *shader);
 
 	for (size_t i = 0; i < positionsN; ++i)
 	{
@@ -1127,7 +1129,7 @@ void GroundLayer::draw()
 		if (!(p->replicas & pos))
 			continue;
 
-		p->setTranslation(pos, shader);
+		p->setTranslation(pos, *shader);
 
 		drawInt();
 	}
@@ -1163,7 +1165,7 @@ void GroundLayer::draw()
 void GroundLayer::drawInt()
 {
 	gl.DrawElements(GL_TRIANGLES, vboCount,
-	                GL_UNSIGNED_INT, (GLvoid*) (p->tiles.frameIdx * p->tiles.bufferFrameSize));
+	                GL_UNSIGNED_INT, (GLvoid*) 0);
 }
 
 void GroundLayer::drawFlashInt()
@@ -1193,15 +1195,14 @@ void ScanRow::draw()
 	if (batchedFlag)
 		return;
 
-	SimpleShader &shader = shState->shaders().simple;
+	ShaderBase *shader;
-	shader.bind();
-	shader.applyViewportProj();
 
-	p->bindAtlas(shader);
+	p->bindShader(shader);
+	p->bindAtlas(*shader);
 
 	VAO::bind(p->tiles.vao);
 
-	p->setTranslation(Normal, shader);
+	p->setTranslation(Normal, *shader);
 
 	for (size_t i = 0; i < positionsN; ++i)
 	{
@@ -1210,7 +1211,7 @@ void ScanRow::draw()
 		if (!(p->replicas & pos))
 			continue;
 
-		p->setTranslation(pos, shader);
+		p->setTranslation(pos, *shader);
 
 		drawInt();
 	}
@@ -1221,7 +1222,7 @@ void ScanRow::draw()
 void ScanRow::drawInt()
 {
 	gl.DrawElements(GL_TRIANGLES, vboBatchCount,
-	                GL_UNSIGNED_INT, (GLvoid*) (vboOffset + p->tiles.frameIdx * p->tiles.bufferFrameSize));
+	                GL_UNSIGNED_INT, (GLvoid*) vboOffset);
 }
 
 void ScanRow::initUpdateZ()