diff --git a/MobileGL/MG_State/GLState/BufferState/BufferObject.cpp b/MobileGL/MG_State/GLState/BufferState/BufferObject.cpp index fd43cae5..8bfe428b 100644 --- a/MobileGL/MG_State/GLState/BufferState/BufferObject.cpp +++ b/MobileGL/MG_State/GLState/BufferState/BufferObject.cpp @@ -67,6 +67,14 @@ namespace MobileGL::MG_State::GLState { } void BufferObject::NotifyContentWrite(SizeT offset, SizeT size) { + if (size == 0) { + // An empty write moves the serial and nothing else, exactly as NotifySubData + // and NotifyFlushMappedRange do: it wrote no byte, so it must not promote an + // undefined store to "has content" - that would cost the next orphaning + // respecification a full-size upload of bytes the application never wrote. + ++m_changeSerial; + return; + } m_hasDefinedContent = true; if (m_resource.IsGpuResident()) { // The write already landed in coherent GPU memory; the backend has no separate @@ -111,7 +119,10 @@ namespace MobileGL::MG_State::GLState { } void BufferObject::Respecify(SizeT size, const void* data) { - ReleaseMemory(); + // The store a live mapping wrote into is about to be replaced, so landing those + // bytes into it would copy a whole mapped range (an adopted arena's map is the + // arena) into storage the next line hands back. + ReleaseMemory(false); RedefineStorage(size); if (data && size > 0) { Memcpy(m_resource.Bytes(), data, size); @@ -136,7 +147,9 @@ namespace MobileGL::MG_State::GLState { } void BufferObject::AllocateImmutableStorage(SizeT size, const void* data, GLbitfield storageFlags) { - ReleaseMemory(); + // Same as Respecify: the bytes a live mapping staged have nowhere to land, the + // store they belong to is being replaced. + ReleaseMemory(false); RedefineStorage(size); if (data) { Memcpy(m_resource.Bytes(), data, size); @@ -190,24 +203,45 @@ namespace MobileGL::MG_State::GLState { m_usage = usage; } - void BufferObject::ReleaseMemory() { + void BufferObject::ReleaseMemory(Bool landStagedWrites) { if (!m_isMapped) return; - if (m_mappingAccess & BufferMappingAccessBit::Write) { // if we wrote to the buffer - // A persistent GPU-resident map wrote straight into coherent GPU memory, so - // there is nothing to copy back and no range to push down on unmap. - if (!m_resource.IsGpuResident() && - !(m_mappingAccess & BufferMappingAccessBit::FlushExplicit)) { // if we didn't flush explicitly - if (!(m_mappingAccess & BufferMappingAccessBit::Persistent)) { - Memcpy(m_resource.Bytes() + m_mappedRange.start, m_stagingData.data() + m_stagingBias, - m_mappedRange.end - m_mappedRange.start); + if (landStagedWrites && + (m_mappingAccess & BufferMappingAccessBit::Write)) { // if we wrote to the buffer + if (!(m_mappingAccess & BufferMappingAccessBit::FlushExplicit)) { // if we didn't flush explicitly + const SizeT mappedLength = m_mappedRange.end - m_mappedRange.start; + if (m_resource.IsGpuResident()) { + // A persistent map of an adopted store wrote straight into coherent + // GPU memory: nothing to copy back, no range to push down. A + // NON-persistent write map is a different thing: the application + // wrote a staging copy (glMapBuffer and glMapBufferRange hand one out + // regardless of where the store lives), and GL requires those bytes + // to be visible to every later command the moment glUnmapBuffer + // returns. Residency used to come only from a coherent persistent + // map, which never has a staging copy, so the copy-back was simply + // skipped for a resident store; residency now also comes from a + // shader storage binding (EnsureGpuResidentStorage at draw time) and + // from large-store adoption (TryAdoptLargeStorage), both of which an + // application then re-initialises through an ordinary map/write/unmap. + // Skipping the copy-back dropped every one of those writes. Land the + // staged bytes through the same route glBufferSubData takes into an + // adopted store - the backend's flush op is for stores it keeps a + // separate copy of and must not run here. + if (!(m_mappingAccess & BufferMappingAccessBit::Persistent)) { + LandBytesIntoResidentStore(m_mappedRange.start, + {m_stagingData.data() + m_stagingBias, mappedLength}); + } + } else { + if (!(m_mappingAccess & BufferMappingAccessBit::Persistent)) { + Memcpy(m_resource.Bytes() + m_mappedRange.start, m_stagingData.data() + m_stagingBias, + mappedLength); + } + NotifyFlushMappedRange(m_mappedRange, m_mappingAccess); } - NotifyFlushMappedRange(m_mappedRange, m_mappingAccess); } - - m_stagingData.clear(); } + m_stagingData.clear(); m_isMapped = false; m_mappingAccess = BufferMappingAccessBit::Null; m_mappedRange = {0, 0}; @@ -227,8 +261,21 @@ namespace MobileGL::MG_State::GLState { MOBILEGL_ASSERT(end <= m_mappedRange.end, "Flush range out of bounds: mappedRange.end (%zu) < end (%zu)", m_mappedRange.end, end); - // FLUSH_EXPLICIT maps are never GPU-resident (only coherent maps are adopted), so - // the staged bytes must be copied into the shadow before the backend reads them. + // A FLUSH_EXPLICIT map can sit on an adopted store: the map itself never adopts + // (only a coherent persistent one does), but a shader storage binding or + // large-store adoption may have made the buffer resident before the map. The + // flushed bytes then take the same landing as any other CPU write into an + // adopted store - a persistent map already wrote them in place and only has + // to publish the change, a non-persistent map staged them and has to land + // them. The backend's flush op is for stores it keeps a separate copy of. + if (m_resource.IsGpuResident()) { + if (m_mappingAccess & BufferMappingAccessBit::Persistent) { + NotifyContentWrite(start, length); + } else { + LandBytesIntoResidentStore(start, {m_stagingData.data() + m_stagingBias + offset, length}); + } + return; + } if (!(m_mappingAccess & BufferMappingAccessBit::Persistent)) { Memcpy(m_resource.Bytes() + start, m_stagingData.data() + m_stagingBias + offset, length); } @@ -284,35 +331,48 @@ namespace MobileGL::MG_State::GLState { data.size, m_size); // An adopted store's Bytes() IS the memory in-flight frames are reading, and - // GL orders a glBufferSubData after those already-submitted reads. A backend - // that can land the bytes on the GPU timeline takes them here, untouched by - // the mapping - the in-place host write below tore the frames still reading - // the old bytes. The bytes are not current in the mapping until the backend's - // ordered copy executes, so reads reconcile through the same gate GPU-written - // buffers use. - if (m_resource.IsGpuResident() && data.size > 0 && g_bufferBackendOps && - g_bufferBackendOps->ResidentSubData) { - g_bufferBackendOps->ResidentSubData(*this, atOffset, data); + // GL orders a glBufferSubData after those already-submitted reads: the write + // has to take the resident landing, never a plain host write into the mapping. + // Shadow-backed stores need none of this: the Memcpy below touches only the + // shadow, and the backend's SubData op does its own ordering against in-flight + // work. + if (m_resource.IsGpuResident()) { + LandBytesIntoResidentStore(atOffset, data); + return; + } + + Memcpy(m_resource.Bytes() + atOffset, data.data, data.size); + NotifyContentWrite(atOffset, data.size); + } + + // A backend that can land the bytes on the GPU timeline takes them here, untouched + // by the mapping - an in-place host write into coherent memory tore the frames + // still reading the old bytes (Minecraft patches LIVE chunk sections this way). + // The bytes are then not current in the mapping until the backend's ordered copy + // executes, so reads reconcile through the same gate GPU-written buffers use. + // + // Without that op the write lands in place, after retiring the GPU writes this store + // is known to be waiting on: a backend that defers work (DirectVulkan's frame command + // buffer) may still be holding a recorded-but-unsubmitted dispatch that GL orders this + // write AFTER, and writing the mapping now would land the bytes underneath that + // dispatch - its increments then execute on top of the newer data and invert the call + // order. That gate only knows about work that WROTE the store (MarkGpuWritten); work + // that merely READS it - a draw sourcing an adopted vertex arena - is not tracked here, + // so a backend without the op still owes the ordering against its own recorded reads. + // NotifyContentWrite on a resident store only bumps the serial: the backend has no + // separate copy to sync, so no transfer op runs. + void BufferObject::LandBytesIntoResidentStore(SizeT offset, DataPtr bytes) { + if (bytes.size > 0 && g_bufferBackendOps && g_bufferBackendOps->ResidentSubData) { + g_bufferBackendOps->ResidentSubData(*this, offset, bytes); m_hasDefinedContent = true; ++m_changeSerial; m_gpuWritePending = true; return; } - // An adopted store's Bytes() IS the memory the GPU reads, and a backend that - // defers work (DirectVulkan's frame command buffer) may still be holding a - // recorded-but-unsubmitted dispatch that GL orders this write AFTER. Writing - // the mapping now would land the bytes underneath that dispatch - its - // increments then execute on top of the newer data and invert the call order. - // Retire the pending GPU writes first, as FillSubData already does. Shadow- - // backed stores need none of this: the Memcpy below touches only the shadow, - // and the backend's SubData op does its own ordering against in-flight work. - if (m_resource.IsGpuResident()) { - SyncGpuWrites(); - } - - Memcpy(m_resource.Bytes() + atOffset, data.data, data.size); - NotifyContentWrite(atOffset, data.size); + SyncGpuWrites(); + Memcpy(m_resource.Bytes() + offset, bytes.data, bytes.size); + NotifyContentWrite(offset, bytes.size); } void BufferObject::FillSubData(DataPtr pattern, SizeT atOffset, SizeT size) { @@ -327,8 +387,13 @@ namespace MobileGL::MG_State::GLState { "Cannot fill data while buffer is non-persistently mapped."); if (size == 0) return; - // An adopted store takes the same GPU-timeline landing as UploadSubData: the - // in-place write below would tear in-flight readers of the mapping. + // An adopted store takes the same landing as UploadSubData: the in-place write + // below would tear in-flight readers of the mapping. The pattern is expanded + // first because the landing takes the final bytes, not a repeat rule - which is + // why only a backend that actually takes them comes through here. Without that + // op the landing would memcpy the expansion into the mapping the loop below + // fills in place anyway, so a whole-arena clear would allocate a whole arena + // for nothing. if (m_resource.IsGpuResident() && g_bufferBackendOps && g_bufferBackendOps->ResidentSubData) { Vector expanded(size); if (pattern.size == 1) { @@ -338,16 +403,13 @@ namespace MobileGL::MG_State::GLState { Memcpy(expanded.data() + at, pattern.data, pattern.size); } } - g_bufferBackendOps->ResidentSubData(*this, atOffset, {expanded.data(), size}); - m_hasDefinedContent = true; - ++m_changeSerial; - m_gpuWritePending = true; + LandBytesIntoResidentStore(atOffset, {expanded.data(), size}); return; } - // A clear is ordered after all earlier GPU writes. Partial clears additionally need the - // retained shadow bytes; whole-store clears need the same synchronization before writing - // an adopted persistent mapping that the GPU may still be accessing. + // A clear is ordered after all earlier GPU writes; partial clears additionally need + // the retained shadow bytes, and a resident store the backend cannot take the bytes + // for is written in place, which needs the same synchronization the landing does. SyncGpuWrites(); Uint8* dst = m_resource.Bytes() + atOffset; @@ -381,22 +443,13 @@ namespace MobileGL::MG_State::GLState { size, m_size); src->SyncGpuWrites(); - // An adopted DESTINATION takes the same GPU-timeline landing as UploadSubData; - // the in-place write below would tear in-flight readers of the mapping. - if (m_resource.IsGpuResident() && size > 0 && g_bufferBackendOps && - g_bufferBackendOps->ResidentSubData) { - g_bufferBackendOps->ResidentSubData(*this, dstOffset, - {src->m_resource.Bytes() + srcOffset, size}); - m_hasDefinedContent = true; - ++m_changeSerial; - m_gpuWritePending = true; - return; - } - // The DESTINATION needs the same ordering as UploadSubData: an adopted store is - // written in place, so pending recorded GPU writes to it must retire before the - // copy lands or they would execute on top of it. + // An adopted DESTINATION takes the same landing as UploadSubData: the in-place + // write below would tear in-flight readers of the mapping, and pending recorded + // GPU writes to it must retire before the copy lands or they would execute on + // top of it. if (m_resource.IsGpuResident()) { - SyncGpuWrites(); + LandBytesIntoResidentStore(dstOffset, {src->m_resource.Bytes() + srcOffset, size}); + return; } Memcpy(m_resource.Bytes() + dstOffset, src->m_resource.Bytes() + srcOffset, size); NotifyContentWrite(dstOffset, size); @@ -433,6 +486,16 @@ namespace MobileGL::MG_State::GLState { if (m_resource.IsGpuResident()) { return true; } + // Adoption releases the CPU shadow, and a live mapping may BE that shadow: a + // persistent map that did not itself adopt (a FLUSH_EXPLICIT one, or a read map) + // handed the application shadow + offset, and GL keeps that pointer valid while + // the buffer is drawn with - which is exactly when this runs, on the storage + // binding walk. Freeing it under the application is a use-after-free, so a mapped + // buffer keeps the shadow model until it is unmapped; the binding that follows + // adopts then. Same rule as TryAdoptLargeStorage. + if (m_isMapped) { + return false; + } if (m_size == 0 || g_bufferBackendOps == nullptr || g_bufferBackendOps->AcquirePersistentMap == nullptr) { return false; } @@ -451,7 +514,20 @@ namespace MobileGL::MG_State::GLState { // The app is about to look at the bytes; a shader may have rewritten them since // the shadow was last authoritative. Also needed for a write map without an // invalidate bit, whose staging copy is seeded from the shadow. - SyncGpuWrites(); + // + // One map shape looks at nothing: a non-persistent write map that discards the + // range it maps gets a staging copy the seeding below skips, so no reader of the + // store exists between here and the unmap. Reconciling an ADOPTED store would + // still cost the backend's full drain-and-wait (its queued landings are made + // visible to the CPU by finishing the pipeline), once per map, on exactly the + // streaming arena the adoption exists to keep cheap. The outstanding-write flag + // stays set, so the first read that DOES look at the bytes still pays for it. + const Bool discardsWhatItMaps = + (access & BufferMappingAccessBit::Write) && !(access & BufferMappingAccessBit::Persistent) && + (access & (BufferMappingAccessBit::InvalidateRange | BufferMappingAccessBit::InvalidateBuffer)); + if (!(m_resource.IsGpuResident() && discardsWhatItMaps)) { + SyncGpuWrites(); + } m_isMapped = true; m_mappingAccess = access; m_mappedRange = range; diff --git a/MobileGL/MG_State/GLState/BufferState/BufferObject.h b/MobileGL/MG_State/GLState/BufferState/BufferObject.h index 583fd0dc..f9674530 100644 --- a/MobileGL/MG_State/GLState/BufferState/BufferObject.h +++ b/MobileGL/MG_State/GLState/BufferState/BufferObject.h @@ -81,7 +81,9 @@ namespace MobileGL { // Contents update of [offset, offset + size) from the shadow. void (*SubData)(BufferObject& bufferObject, SizeT offset, SizeT size) = nullptr; // Contents update of an ADOPTED (GPU-resident) store. `data` holds the app's - // bytes; the frontend has NOT touched the resident mapping. GL orders a + // bytes, valid for the duration of the call only (a write map's staging + // store is freed the moment the unmap that lands it returns); the frontend + // has NOT touched the resident mapping. GL orders a // glBufferSubData after already-submitted GPU reads of the store, and an // in-place host write into the coherent mapping tears the frames still // reading the old bytes (Minecraft patches LIVE chunk sections this way - @@ -157,9 +159,14 @@ namespace MobileGL { // Adopt backend host-visible coherent GPU storage as the source of truth // (used for GPU-written targets like transform feedback capture, so // MapBuffer/GetBufferSubData read real GPU results). No-op when already - // resident or when the backend declines. + // resident, while the buffer is mapped (adoption releases the shadow a + // mapping may have handed the application), or when the backend declines. Bool EnsureGpuResidentStorage(); - void ReleaseMemory(); + // Unmap. A write map's staged bytes land in the store on the way out, unless + // the caller is about to replace that store (a respecification) and passes + // false - landing them there would copy a whole mapped range into storage + // being handed back on the next line. + void ReleaseMemory(Bool landStagedWrites = true); void FlushMemoryRange(SizeT offset, SizeT length); // Pushes the persistently-mapped write range to the backend; called by @@ -234,6 +241,12 @@ namespace MobileGL { // so this only bumps the change serial; otherwise it dispatches a backend // SubData transfer to sync the backend's separate GPU copy. void NotifyContentWrite(SizeT offset, SizeT size); + // The one route CPU-sourced bytes take into an ADOPTED (GPU-resident) store: + // glBufferSubData, a buffer clear, a buffer copy, and the landing of a + // non-persistent write map at unmap / explicit flush all go through it, so + // the routes cannot drift apart again. Carries no mapping asserts on + // purpose - the unmap landing runs while the buffer is still mapped. + void LandBytesIntoResidentStore(SizeT offset, DataPtr bytes); static Uint64 AllocateLifetimeId(); diff --git a/MobileGL/MG_Test/Buffer/BufferTest.cpp b/MobileGL/MG_Test/Buffer/BufferTest.cpp index 61cdc560..0db6d477 100644 --- a/MobileGL/MG_Test/Buffer/BufferTest.cpp +++ b/MobileGL/MG_Test/Buffer/BufferTest.cpp @@ -1568,6 +1568,15 @@ namespace { int respecifyCalls = 0; int flushCalls = 0; Bool provideMap = true; // false => backend declines, exercising the shadow fallback + // Only recorded by the variant of the ops table that offers ResidentSubData: the + // bytes a CPU write handed the backend for a GPU-ordered landing into an adopted + // store, held back from `gpu` until a readback "retires" them. + struct ResidentWrite { + SizeT offset = 0; + Vector bytes; + }; + Vector residentWrites; + int readbackCalls = 0; }; ZeroCopyMockBackend* g_zeroCopyMock = nullptr; @@ -1605,6 +1614,39 @@ namespace { .AcquirePersistentMap = ZeroCopyMock_AcquirePersistentMap, }; + // The same backend with the GPU-ordered landing ops a staging-ring backend offers: a + // CPU write into an adopted store is queued (the mapping is NOT written through), and + // a readback is what lands the queue before the application reads. + void ZeroCopyMock_ResidentSubData(MG_State::GLState::BufferObject&, SizeT offset, DataPtr data) { + if (!g_zeroCopyMock) return; + auto& write = g_zeroCopyMock->residentWrites.emplace_back(); + write.offset = offset; + const auto* bytes = static_cast(data.data); + write.bytes.assign(bytes, bytes + data.size); + } + void ZeroCopyMock_ReadbackFromGpu(MG_State::GLState::BufferObject&) { + if (!g_zeroCopyMock) return; + ++g_zeroCopyMock->readbackCalls; + for (const auto& write : g_zeroCopyMock->residentWrites) { + // Reported, not asserted: an ASSERT here would return out of the readback and + // leave the remaining landings unapplied, which reads as a different failure. + EXPECT_LE(write.offset + write.bytes.size(), g_zeroCopyMock->gpu.size()); + if (write.offset + write.bytes.size() > g_zeroCopyMock->gpu.size()) continue; + Memcpy(g_zeroCopyMock->gpu.data() + write.offset, write.bytes.data(), write.bytes.size()); + } + g_zeroCopyMock->residentWrites.clear(); + } + + const MG_State::GLState::BufferBackendOps kResidentSubDataMockOps = { + .Respecify = ZeroCopyMock_Respecify, + .SubData = ZeroCopyMock_SubData, + .ResidentSubData = ZeroCopyMock_ResidentSubData, + .FlushMappedRange = ZeroCopyMock_Flush, + .OnDestroy = ZeroCopyMock_OnDestroy, + .AcquirePersistentMap = ZeroCopyMock_AcquirePersistentMap, + .ReadbackFromGpu = ZeroCopyMock_ReadbackFromGpu, + }; + struct ScopedBackendOps { explicit ScopedBackendOps(const MG_State::GLState::BufferBackendOps* ops) { MG_State::GLState::SetBufferBackendOps(ops); @@ -2074,3 +2116,861 @@ TEST_F(BufferTest, RedefiningANonAdoptedBufferIsUnchanged) { g_zeroCopyMock = nullptr; } + +// --------------------------------------------------------------------------- +// A NON-persistent write map of an ADOPTED store. glMapBuffer / glMapBufferRange hand +// the application a staging copy regardless of where the store lives, and GL requires +// the bytes it wrote there to be visible to every later command once glUnmapBuffer +// returns. Residency used to come only from a coherent persistent map - which writes +// in place and never has a staging copy - so the unmap simply skipped the copy-back +// for a resident store. Residency now also comes from a shader storage binding +// (EnsureGpuResidentStorage at draw time) and from large-store adoption, both of which +// an application then re-initialises through an ordinary map/write/unmap: the +// conformance suite re-seeds every SSBO that way before each draw, and every re-seed +// after the first draw was dropped on the floor. These pin the landing for each map +// shape, on the backend that writes the coherent mapping in place and on the one that +// takes the bytes for a GPU-ordered landing, plus the shadow path as the control. +namespace { + constexpr SizeT kAdoptedInts = 16; + + // A buffer of kAdoptedInts sequential ints, adopted by the mock backend exactly as an + // SSBO binding does at draw time. The per-write counters are zeroed afterwards so a + // test only sees the traffic of the map it makes. + SharedPtr MakeAdoptedBuffer(ZeroCopyMockBackend& mock, GLenum target, + GLuint& buffer) { + GenBuffers(1, &buffer); + BindBuffer(target, buffer); + Vector initial(kAdoptedInts); + for (SizeT i = 0; i < kAdoptedInts; ++i) initial[i] = static_cast(i); + BufferData(target, static_cast(kAdoptedInts * sizeof(GLint)), initial.data(), + GL_DYNAMIC_DRAW); + EXPECT_EQ(GetError(), GL_NO_ERROR); + auto bufferObject = MG_State::pGLContext->GetBufferObject(buffer); + EXPECT_NE(bufferObject, nullptr); + if (bufferObject == nullptr) return nullptr; + EXPECT_TRUE(bufferObject->EnsureGpuResidentStorage()); + EXPECT_TRUE(bufferObject->IsBackendPersistentMapped()); + EXPECT_EQ(static_cast(bufferObject->MappedData()), static_cast(mock.gpu.data())); + mock.subDataCalls = 0; + mock.flushCalls = 0; + mock.respecifyCalls = 0; + return bufferObject; + } + + const GLint* GpuInts(const ZeroCopyMockBackend& mock) { + return reinterpret_cast(mock.gpu.data()); + } +} // namespace + +TEST_F(BufferTest, ANonPersistentReadWriteRangeMapOfAnAdoptedStoreLandsAtUnmap) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_SHADER_STORAGE_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + const Uint64 baseSerial = bufferObject->GetChangeSerial(); + + // The conformance suite's shape: the whole store, READ|WRITE, then a full rewrite. + auto* mapped = static_cast(bufferObject->AcquireMemoryRange( + {0, kAdoptedInts * sizeof(GLint)}, BufferMappingAccessBit::Read | BufferMappingAccessBit::Write)); + ASSERT_NE(mapped, nullptr); + // A non-persistent map is a staging copy, seeded from the adopted store... + EXPECT_NE(static_cast(mapped), static_cast(mock.gpu.data())); + for (SizeT i = 0; i < kAdoptedInts; ++i) EXPECT_EQ(mapped[i], static_cast(i)); + for (SizeT i = 0; i < kAdoptedInts; ++i) mapped[i] = 1000 + static_cast(i); + // ...that the store does not see until the unmap. + EXPECT_EQ(GpuInts(mock)[0], 0); + bufferObject->ReleaseMemory(); + + EXPECT_FALSE(bufferObject->IsMapped()); + EXPECT_TRUE(bufferObject->IsBackendPersistentMapped()); + for (SizeT i = 0; i < kAdoptedInts; ++i) { + EXPECT_EQ(GpuInts(mock)[i], 1000 + static_cast(i)) << "int " << i; + } + EXPECT_EQ(std::memcmp(bufferObject->MappedData(), mock.gpu.data(), mock.gpu.size()), 0); + // The landing publishes the change for cached consumers... + EXPECT_GT(bufferObject->GetChangeSerial(), baseSerial); + // ...but dispatches no transfer op: the backend keeps no separate copy of an + // adopted store, and its flush op would only upload the mapping onto itself. + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_EQ(mock.acquireMapCalls, 1); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +TEST_F(BufferTest, GlMapBufferWriteOnlyAndReadWriteOfAnAdoptedStoreLandAtUnmap) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_SHADER_STORAGE_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + + // glMapBuffer(GL_WRITE_ONLY): the staging copy is still seeded (no invalidate bit), + // so a partial write keeps the untouched ints. + Uint64 serial = bufferObject->GetChangeSerial(); + auto* writeOnly = static_cast(bufferObject->AcquireMemory(true, false, true)); + ASSERT_NE(writeOnly, nullptr); + EXPECT_NE(static_cast(writeOnly), static_cast(mock.gpu.data())); + writeOnly[0] = 100; + writeOnly[1] = 200; + bufferObject->ReleaseMemory(); + EXPECT_EQ(GpuInts(mock)[0], 100); + EXPECT_EQ(GpuInts(mock)[1], 200); + EXPECT_EQ(GpuInts(mock)[2], 2); + EXPECT_EQ(GpuInts(mock)[kAdoptedInts - 1], static_cast(kAdoptedInts - 1)); + EXPECT_GT(bufferObject->GetChangeSerial(), serial); + + // glMapBuffer(GL_READ_WRITE): reads see the previous landing, and the next one lands too. + serial = bufferObject->GetChangeSerial(); + auto* readWrite = static_cast(bufferObject->AcquireMemory(true, true, true)); + ASSERT_NE(readWrite, nullptr); + EXPECT_EQ(readWrite[0], 100); + EXPECT_EQ(readWrite[1], 200); + readWrite[2] = 300; + bufferObject->ReleaseMemory(); + EXPECT_EQ(GpuInts(mock)[0], 100); + EXPECT_EQ(GpuInts(mock)[1], 200); + EXPECT_EQ(GpuInts(mock)[2], 300); + EXPECT_GT(bufferObject->GetChangeSerial(), serial); + + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_TRUE(bufferObject->IsBackendPersistentMapped()); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +TEST_F(BufferTest, AWriteMapInvalidatingAnAdoptedStoreLandsTheWholeRangeAtUnmap) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_SHADER_STORAGE_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + const Uint64 baseSerial = bufferObject->GetChangeSerial(); + + auto* mapped = static_cast(bufferObject->AcquireMemoryRange( + {0, kAdoptedInts * sizeof(GLint)}, BufferMappingAccessBit::Write | BufferMappingAccessBit::InvalidateBuffer)); + ASSERT_NE(mapped, nullptr); + EXPECT_NE(static_cast(mapped), static_cast(mock.gpu.data())); + // The whole range is undefined by contract, so the application rewrites all of it. + for (SizeT i = 0; i < kAdoptedInts; ++i) mapped[i] = -static_cast(i) - 1; + bufferObject->ReleaseMemory(); + + for (SizeT i = 0; i < kAdoptedInts; ++i) { + EXPECT_EQ(GpuInts(mock)[i], -static_cast(i) - 1) << "int " << i; + } + EXPECT_GT(bufferObject->GetChangeSerial(), baseSerial); + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +// A range map at an offset off the alignment grid: the staging store is biased by the +// offset's phase (see AcquireMemoryRange), and the landing has to read from the biased +// start and write to the mapped offset - not from data(), not to 0. +TEST_F(BufferTest, ARangeMapAtAnUnalignedOffsetOfAnAdoptedStoreLandsInPlace) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_ARRAY_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + const Uint64 baseSerial = bufferObject->GetChangeSerial(); + + // Ints 3..6, i.e. byte offset 12 - inside the first alignment, so the bias is non-zero. + constexpr SizeT kFirst = 3; + constexpr SizeT kCount = 4; + const Range1D range{kFirst * sizeof(GLint), (kFirst + kCount) * sizeof(GLint)}; + ASSERT_NE(range.start % MG_State::GLState::MIN_MAP_BUFFER_ALIGNMENT, 0u); + auto* mapped = static_cast(bufferObject->AcquireMemoryRange(range, BufferMappingAccessBit::Write)); + ASSERT_NE(mapped, nullptr); + // Seeded from the right place... + for (SizeT i = 0; i < kCount; ++i) EXPECT_EQ(mapped[i], static_cast(kFirst + i)); + for (SizeT i = 0; i < kCount; ++i) mapped[i] = 500 + static_cast(i); + bufferObject->ReleaseMemory(); + + // ...and landed in the right place, with everything outside the range untouched. + for (SizeT i = 0; i < kAdoptedInts; ++i) { + const GLint expected = (i >= kFirst && i < kFirst + kCount) ? 500 + static_cast(i - kFirst) + : static_cast(i); + EXPECT_EQ(GpuInts(mock)[i], expected) << "int " << i; + } + EXPECT_GT(bufferObject->GetChangeSerial(), baseSerial); + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +// FLUSH_EXPLICIT on an adopted store: only the flushed bytes land, at the flush, and the +// unmap lands nothing more - the application promised to flush what it wanted kept. +TEST_F(BufferTest, AnExplicitFlushOfAWriteMapOfAnAdoptedStoreLandsOnlyTheFlushedBytes) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_SHADER_STORAGE_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + const Uint64 baseSerial = bufferObject->GetChangeSerial(); + + // Ints 2..13 mapped (offset 8, off the grid again), all of them rewritten... + constexpr SizeT kFirst = 2; + constexpr SizeT kCount = 12; + const Range1D range{kFirst * sizeof(GLint), (kFirst + kCount) * sizeof(GLint)}; + auto* mapped = static_cast(bufferObject->AcquireMemoryRange( + range, BufferMappingAccessBit::Write | BufferMappingAccessBit::FlushExplicit)); + ASSERT_NE(mapped, nullptr); + for (SizeT i = 0; i < kCount; ++i) mapped[i] = 700 + static_cast(i); + // ...but only ints 5..8 (map-relative ints 3..6) flushed. + constexpr SizeT kFlushFirst = 3; + constexpr SizeT kFlushCount = 4; + bufferObject->FlushMemoryRange(kFlushFirst * sizeof(GLint), kFlushCount * sizeof(GLint)); + const Uint64 flushSerial = bufferObject->GetChangeSerial(); + EXPECT_GT(flushSerial, baseSerial); + EXPECT_EQ(mock.flushCalls, 0); + + auto expectOnlyFlushedBytesLanded = [&](const char* when) { + for (SizeT i = 0; i < kAdoptedInts; ++i) { + const Bool flushed = i >= kFirst + kFlushFirst && i < kFirst + kFlushFirst + kFlushCount; + const GLint expected = flushed ? 700 + static_cast(i - kFirst) : static_cast(i); + EXPECT_EQ(GpuInts(mock)[i], expected) << when << ": int " << i; + } + }; + expectOnlyFlushedBytesLanded("after the flush"); + + bufferObject->ReleaseMemory(); + expectOnlyFlushedBytesLanded("after the unmap"); + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_TRUE(bufferObject->IsBackendPersistentMapped()); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +// The other kind of backend: one that takes the bytes for a GPU-ordered landing instead +// of letting the frontend write the coherent mapping in place. The unmap hands it the +// mapped offset and the bias-adjusted bytes, leaves the mapping alone, and marks a GPU +// write outstanding so the next read reconciles through the readback. +TEST_F(BufferTest, ABackendWithAResidentSubDataOpTakesTheUnmappedBytesForAGpuOrderedLanding) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kResidentSubDataMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_SHADER_STORAGE_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + const Uint64 baseSerial = bufferObject->GetChangeSerial(); + + constexpr SizeT kFirst = 3; + constexpr SizeT kCount = 5; + const Range1D range{kFirst * sizeof(GLint), (kFirst + kCount) * sizeof(GLint)}; + auto* mapped = static_cast(bufferObject->AcquireMemoryRange( + range, BufferMappingAccessBit::Read | BufferMappingAccessBit::Write)); + ASSERT_NE(mapped, nullptr); + for (SizeT i = 0; i < kCount; ++i) mapped[i] = 900 + static_cast(i); + bufferObject->ReleaseMemory(); + + // The op got exactly the mapped range's bytes at the mapped offset... + ASSERT_EQ(mock.residentWrites.size(), 1u); + EXPECT_EQ(mock.residentWrites[0].offset, range.start); + ASSERT_EQ(mock.residentWrites[0].bytes.size(), kCount * sizeof(GLint)); + const auto* handed = reinterpret_cast(mock.residentWrites[0].bytes.data()); + for (SizeT i = 0; i < kCount; ++i) EXPECT_EQ(handed[i], 900 + static_cast(i)) << "int " << i; + // ...the mapping itself was not written through... + for (SizeT i = 0; i < kAdoptedInts; ++i) EXPECT_EQ(GpuInts(mock)[i], static_cast(i)) << "int " << i; + EXPECT_GT(bufferObject->GetChangeSerial(), baseSerial); + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_EQ(mock.readbackCalls, 0); + + // ...and the pending flag makes the next read pull the landing back first. + const auto* readBack = static_cast(bufferObject->AcquireMemory(false, true, false)); + EXPECT_EQ(mock.readbackCalls, 1); + EXPECT_TRUE(mock.residentWrites.empty()); + for (SizeT i = 0; i < kAdoptedInts; ++i) { + const GLint expected = (i >= kFirst && i < kFirst + kCount) ? 900 + static_cast(i - kFirst) + : static_cast(i); + EXPECT_EQ(readBack[i], expected) << "int " << i; + } + // A second read has nothing outstanding to reconcile. + bufferObject->AcquireMemory(false, true, false); + EXPECT_EQ(mock.readbackCalls, 1); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +TEST_F(BufferTest, ABackendWithAResidentSubDataOpTakesAnExplicitlyFlushedRangeTheSameWay) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kResidentSubDataMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_SHADER_STORAGE_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + + constexpr SizeT kFirst = 2; + constexpr SizeT kCount = 8; + const Range1D range{kFirst * sizeof(GLint), (kFirst + kCount) * sizeof(GLint)}; + auto* mapped = static_cast(bufferObject->AcquireMemoryRange( + range, BufferMappingAccessBit::Write | BufferMappingAccessBit::FlushExplicit)); + ASSERT_NE(mapped, nullptr); + for (SizeT i = 0; i < kCount; ++i) mapped[i] = 800 + static_cast(i); + constexpr SizeT kFlushFirst = 5; + constexpr SizeT kFlushCount = 2; + bufferObject->FlushMemoryRange(kFlushFirst * sizeof(GLint), kFlushCount * sizeof(GLint)); + + ASSERT_EQ(mock.residentWrites.size(), 1u); + EXPECT_EQ(mock.residentWrites[0].offset, (kFirst + kFlushFirst) * sizeof(GLint)); + ASSERT_EQ(mock.residentWrites[0].bytes.size(), kFlushCount * sizeof(GLint)); + const auto* handed = reinterpret_cast(mock.residentWrites[0].bytes.data()); + EXPECT_EQ(handed[0], 800 + static_cast(kFlushFirst)); + EXPECT_EQ(handed[1], 800 + static_cast(kFlushFirst + 1)); + EXPECT_EQ(mock.flushCalls, 0); + + // The unmap of a FLUSH_EXPLICIT map adds nothing. + bufferObject->ReleaseMemory(); + EXPECT_EQ(mock.residentWrites.size(), 1u); + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +// The CTS idiom end to end through the GL entry points: an SSBO made resident by a +// draw, re-seeded with glMapBufferRange(READ|WRITE) + glUnmapBuffer. +TEST_F(BufferTest, MapBufferRangeAndUnmapBufferReseedAnAdoptedShaderStorageBuffer) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_SHADER_STORAGE_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + + for (GLint pass = 1; pass <= 3; ++pass) { + auto* mapped = static_cast( + MapBufferRange(GL_SHADER_STORAGE_BUFFER, 0, static_cast(kAdoptedInts * sizeof(GLint)), + GL_MAP_READ_BIT | GL_MAP_WRITE_BIT)); + ASSERT_NE(mapped, nullptr); + ASSERT_EQ(GetError(), GL_NO_ERROR); + for (SizeT i = 0; i < kAdoptedInts; ++i) mapped[i] = pass * 100 + static_cast(i); + EXPECT_TRUE(UnmapBuffer(GL_SHADER_STORAGE_BUFFER)); + ASSERT_EQ(GetError(), GL_NO_ERROR); + for (SizeT i = 0; i < kAdoptedInts; ++i) { + EXPECT_EQ(GpuInts(mock)[i], pass * 100 + static_cast(i)) << "pass " << pass << " int " << i; + } + } + EXPECT_TRUE(bufferObject->IsBackendPersistentMapped()); + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +// The control: a store the backend declined to adopt keeps the shadow model exactly as +// before - the staging copy is written back into the shadow and the backend's flush op +// carries the range down. +TEST_F(BufferTest, ANonPersistentWriteMapOfAShadowBackedStoreStillFlushesThroughTheBackend) { + ZeroCopyMockBackend mock; + mock.provideMap = false; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + + GLuint buffer = 0; + GenBuffers(1, &buffer); + BindBuffer(GL_SHADER_STORAGE_BUFFER, buffer); + Vector initial(kAdoptedInts); + for (SizeT i = 0; i < kAdoptedInts; ++i) initial[i] = static_cast(i); + BufferData(GL_SHADER_STORAGE_BUFFER, static_cast(kAdoptedInts * sizeof(GLint)), initial.data(), + GL_DYNAMIC_DRAW); + ASSERT_EQ(GetError(), GL_NO_ERROR); + auto bufferObject = MG_State::pGLContext->GetBufferObject(buffer); + ASSERT_NE(bufferObject, nullptr); + EXPECT_FALSE(bufferObject->EnsureGpuResidentStorage()); + EXPECT_FALSE(bufferObject->IsBackendPersistentMapped()); + mock.flushCalls = 0; + mock.subDataCalls = 0; + const Uint64 baseSerial = bufferObject->GetChangeSerial(); + + constexpr SizeT kFirst = 3; + constexpr SizeT kCount = 4; + const Range1D range{kFirst * sizeof(GLint), (kFirst + kCount) * sizeof(GLint)}; + auto* mapped = static_cast(bufferObject->AcquireMemoryRange(range, BufferMappingAccessBit::Write)); + ASSERT_NE(mapped, nullptr); + for (SizeT i = 0; i < kCount; ++i) mapped[i] = 600 + static_cast(i); + bufferObject->ReleaseMemory(); + + EXPECT_EQ(mock.flushCalls, 1); + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_GT(bufferObject->GetChangeSerial(), baseSerial); + const auto* shadow = reinterpret_cast(bufferObject->MappedData()); + for (SizeT i = 0; i < kAdoptedInts; ++i) { + const GLint expected = (i >= kFirst && i < kFirst + kCount) ? 600 + static_cast(i - kFirst) + : static_cast(i); + EXPECT_EQ(shadow[i], expected) << "int " << i; + } + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +// --------------------------------------------------------------------------- +// The other three CPU-sourced writes that share the unmap landing's route into an +// adopted store - glBufferSubData, a clear, and a copy - on both kinds of backend: the +// one that lets the frontend write the coherent mapping in place, and the one that takes +// the bytes for a GPU-ordered landing, where the offset it is handed is the only thing +// deciding where they end up. +TEST_F(BufferTest, GlBufferSubDataIntoAnAdoptedStoreLandsInPlaceWithoutABackendTransfer) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_ARRAY_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + + constexpr SizeT kFirst = 4; + const GLint updated[] = {70, 71, 72}; + BufferSubData(GL_ARRAY_BUFFER, static_cast(kFirst * sizeof(GLint)), sizeof(updated), updated); + ASSERT_EQ(GetError(), GL_NO_ERROR); + + for (SizeT i = 0; i < kAdoptedInts; ++i) { + const GLint expected = (i >= kFirst && i < kFirst + 3) ? updated[i - kFirst] : static_cast(i); + EXPECT_EQ(GpuInts(mock)[i], expected) << "int " << i; + } + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_EQ(mock.flushCalls, 0); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +TEST_F(BufferTest, ABackendWithAResidentSubDataOpTakesAGlBufferSubDataAtItsOffset) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kResidentSubDataMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_ARRAY_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + + constexpr SizeT kFirst = 4; + const GLint updated[] = {70, 71, 72}; + BufferSubData(GL_ARRAY_BUFFER, static_cast(kFirst * sizeof(GLint)), sizeof(updated), updated); + ASSERT_EQ(GetError(), GL_NO_ERROR); + + ASSERT_EQ(mock.residentWrites.size(), 1u); + EXPECT_EQ(mock.residentWrites[0].offset, kFirst * sizeof(GLint)); + ASSERT_EQ(mock.residentWrites[0].bytes.size(), sizeof(updated)); + EXPECT_EQ(std::memcmp(mock.residentWrites[0].bytes.data(), updated, sizeof(updated)), 0); + // The mapping itself is left alone until the backend's ordered copy runs. + for (SizeT i = 0; i < kAdoptedInts; ++i) EXPECT_EQ(GpuInts(mock)[i], static_cast(i)) << "int " << i; + + const auto* readBack = static_cast(bufferObject->AcquireMemory(false, true, false)); + EXPECT_EQ(mock.readbackCalls, 1); + for (SizeT i = 0; i < kAdoptedInts; ++i) { + const GLint expected = (i >= kFirst && i < kFirst + 3) ? updated[i - kFirst] : static_cast(i); + EXPECT_EQ(readBack[i], expected) << "int " << i; + } + EXPECT_EQ(mock.subDataCalls, 0); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +TEST_F(BufferTest, GlClearBufferSubDataRepeatsItsPatternThroughAnAdoptedStoreInPlace) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_SHADER_STORAGE_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + + // A four-byte pattern, so the repeat - not a memset - is what fills the range. + constexpr SizeT kFirst = 5; + constexpr SizeT kCount = 6; + const GLint value = 0x0A0B0C0D; + ClearBufferSubData(GL_SHADER_STORAGE_BUFFER, GL_R32I, static_cast(kFirst * sizeof(GLint)), + static_cast(kCount * sizeof(GLint)), GL_RED_INTEGER, GL_INT, &value); + ASSERT_EQ(GetError(), GL_NO_ERROR); + + for (SizeT i = 0; i < kAdoptedInts; ++i) { + const GLint expected = (i >= kFirst && i < kFirst + kCount) ? value : static_cast(i); + EXPECT_EQ(GpuInts(mock)[i], expected) << "int " << i; + } + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_EQ(mock.flushCalls, 0); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +TEST_F(BufferTest, ABackendWithAResidentSubDataOpTakesTheExpandedClearPattern) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kResidentSubDataMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_SHADER_STORAGE_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + + constexpr SizeT kFirst = 5; + constexpr SizeT kCount = 6; + const GLint value = 0x0A0B0C0D; + ClearBufferSubData(GL_SHADER_STORAGE_BUFFER, GL_R32I, static_cast(kFirst * sizeof(GLint)), + static_cast(kCount * sizeof(GLint)), GL_RED_INTEGER, GL_INT, &value); + ASSERT_EQ(GetError(), GL_NO_ERROR); + + // The backend takes the FINAL bytes, so the pattern arrives already repeated. + ASSERT_EQ(mock.residentWrites.size(), 1u); + EXPECT_EQ(mock.residentWrites[0].offset, kFirst * sizeof(GLint)); + ASSERT_EQ(mock.residentWrites[0].bytes.size(), kCount * sizeof(GLint)); + const auto* handed = reinterpret_cast(mock.residentWrites[0].bytes.data()); + for (SizeT i = 0; i < kCount; ++i) EXPECT_EQ(handed[i], value) << "int " << i; + for (SizeT i = 0; i < kAdoptedInts; ++i) EXPECT_EQ(GpuInts(mock)[i], static_cast(i)) << "int " << i; + + const auto* readBack = static_cast(bufferObject->AcquireMemory(false, true, false)); + EXPECT_EQ(mock.readbackCalls, 1); + for (SizeT i = 0; i < kAdoptedInts; ++i) { + const GLint expected = (i >= kFirst && i < kFirst + kCount) ? value : static_cast(i); + EXPECT_EQ(readBack[i], expected) << "int " << i; + } + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +namespace { + // A plain (never adopted) buffer of kAdoptedInts ints, each `bias` above its index, + // bound to `target` as the source of a copy. + GLuint MakeCopySource(GLenum target, GLint bias) { + GLuint buffer = 0; + GenBuffers(1, &buffer); + BindBuffer(target, buffer); + Vector bytes(kAdoptedInts); + for (SizeT i = 0; i < kAdoptedInts; ++i) bytes[i] = bias + static_cast(i); + BufferData(target, static_cast(kAdoptedInts * sizeof(GLint)), bytes.data(), GL_STATIC_DRAW); + EXPECT_EQ(GetError(), GL_NO_ERROR); + return buffer; + } +} // namespace + +TEST_F(BufferTest, GlCopyBufferSubDataIntoAnAdoptedStoreLandsAtTheDestinationOffset) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + GLuint destination = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_COPY_WRITE_BUFFER, destination); + ASSERT_NE(bufferObject, nullptr); + const GLuint source = MakeCopySource(GL_COPY_READ_BUFFER, 900); + + // Deliberately different source and destination offsets: only the destination one + // may decide where the bytes land. + constexpr SizeT kSrcFirst = 1; + constexpr SizeT kDstFirst = 6; + constexpr SizeT kCount = 3; + CopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, static_cast(kSrcFirst * sizeof(GLint)), + static_cast(kDstFirst * sizeof(GLint)), + static_cast(kCount * sizeof(GLint))); + ASSERT_EQ(GetError(), GL_NO_ERROR); + + for (SizeT i = 0; i < kAdoptedInts; ++i) { + const GLint expected = (i >= kDstFirst && i < kDstFirst + kCount) + ? 900 + static_cast(kSrcFirst + i - kDstFirst) + : static_cast(i); + EXPECT_EQ(GpuInts(mock)[i], expected) << "int " << i; + } + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_EQ(mock.flushCalls, 0); + + GLuint toDelete[] = {destination, source}; + DeleteBuffers(2, toDelete); + g_zeroCopyMock = nullptr; +} + +TEST_F(BufferTest, ABackendWithAResidentSubDataOpTakesACopyAtTheDestinationOffset) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kResidentSubDataMockOps); + GLuint destination = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_COPY_WRITE_BUFFER, destination); + ASSERT_NE(bufferObject, nullptr); + const GLuint source = MakeCopySource(GL_COPY_READ_BUFFER, 900); + + constexpr SizeT kSrcFirst = 1; + constexpr SizeT kDstFirst = 6; + constexpr SizeT kCount = 3; + CopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, static_cast(kSrcFirst * sizeof(GLint)), + static_cast(kDstFirst * sizeof(GLint)), + static_cast(kCount * sizeof(GLint))); + ASSERT_EQ(GetError(), GL_NO_ERROR); + + ASSERT_EQ(mock.residentWrites.size(), 1u); + EXPECT_EQ(mock.residentWrites[0].offset, kDstFirst * sizeof(GLint)); + ASSERT_EQ(mock.residentWrites[0].bytes.size(), kCount * sizeof(GLint)); + const auto* handed = reinterpret_cast(mock.residentWrites[0].bytes.data()); + for (SizeT i = 0; i < kCount; ++i) { + EXPECT_EQ(handed[i], 900 + static_cast(kSrcFirst + i)) << "int " << i; + } + for (SizeT i = 0; i < kAdoptedInts; ++i) EXPECT_EQ(GpuInts(mock)[i], static_cast(i)) << "int " << i; + + GLuint toDelete[] = {destination, source}; + DeleteBuffers(2, toDelete); + g_zeroCopyMock = nullptr; +} + +// A PERSISTENT map of an adopted store is the one write shape that needs no landing at +// all: it wrote the coherent mapping in place. Its explicit flush therefore publishes the +// change and dispatches nothing - not the backend's flush op (whose upload would be the +// mapping onto itself) and not the resident landing op (whose bytes are already there). +TEST_F(BufferTest, AnExplicitFlushOfAPersistentMapOfAnAdoptedStoreOnlyPublishesTheChange) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kResidentSubDataMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_SHADER_STORAGE_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + const Uint64 baseSerial = bufferObject->GetChangeSerial(); + + constexpr SizeT kFirst = 2; + constexpr SizeT kCount = 8; + const Range1D range{kFirst * sizeof(GLint), (kFirst + kCount) * sizeof(GLint)}; + auto* mapped = static_cast(bufferObject->AcquireMemoryRange( + range, BufferMappingAccessBit::Write | BufferMappingAccessBit::Persistent | + BufferMappingAccessBit::FlushExplicit)); + ASSERT_NE(mapped, nullptr); + // The application writes the store itself: the map IS the adopted memory. + EXPECT_EQ(static_cast(mapped), static_cast(mock.gpu.data() + range.start)); + for (SizeT i = 0; i < kCount; ++i) mapped[i] = 400 + static_cast(i); + + constexpr SizeT kFlushFirst = 3; + constexpr SizeT kFlushCount = 2; + bufferObject->FlushMemoryRange(kFlushFirst * sizeof(GLint), kFlushCount * sizeof(GLint)); + EXPECT_GT(bufferObject->GetChangeSerial(), baseSerial); + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_TRUE(mock.residentWrites.empty()); + EXPECT_TRUE(bufferObject->HasDefinedContent()); + // Every byte the map wrote is in the store, flushed or not - it was written there. + for (SizeT i = 0; i < kCount; ++i) { + EXPECT_EQ(GpuInts(mock)[kFirst + i], 400 + static_cast(i)) << "int " << i; + } + + const Uint64 flushSerial = bufferObject->GetChangeSerial(); + bufferObject->ReleaseMemory(); + EXPECT_EQ(bufferObject->GetChangeSerial(), flushSerial); // the unmap adds nothing + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_TRUE(mock.residentWrites.empty()); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +// A flush of nothing wrote no byte, so it may not report the store as written: an +// orphaning respecification prices a "has content" store as a full-size upload. +TEST_F(BufferTest, AZeroLengthExplicitFlushOfAnAdoptedStoreLeavesItUndefined) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + + GLuint buffer = 0; + GenBuffers(1, &buffer); + BindBuffer(GL_SHADER_STORAGE_BUFFER, buffer); + BufferData(GL_SHADER_STORAGE_BUFFER, static_cast(kAdoptedInts * sizeof(GLint)), nullptr, + GL_DYNAMIC_DRAW); + ASSERT_EQ(GetError(), GL_NO_ERROR); + auto bufferObject = MG_State::pGLContext->GetBufferObject(buffer); + ASSERT_NE(bufferObject, nullptr); + ASSERT_FALSE(bufferObject->HasDefinedContent()); + ASSERT_TRUE(bufferObject->EnsureGpuResidentStorage()); + const Uint64 baseSerial = bufferObject->GetChangeSerial(); + + auto* mapped = bufferObject->AcquireMemoryRange({0, kAdoptedInts * sizeof(GLint)}, + BufferMappingAccessBit::Write | + BufferMappingAccessBit::Persistent | + BufferMappingAccessBit::FlushExplicit); + ASSERT_NE(mapped, nullptr); + bufferObject->FlushMemoryRange(0, 0); + EXPECT_GT(bufferObject->GetChangeSerial(), baseSerial); + EXPECT_FALSE(bufferObject->HasDefinedContent()); + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + bufferObject->ReleaseMemory(); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +// A write map that discards the range it maps reads nothing of the store: its staging +// copy is not seeded from it. Reconciling an adopted store at map time would run the +// backend's drain-and-wait for no reader, once per map, on the streaming arena the +// adoption exists to keep cheap - so it is deferred, not dropped: the first map that DOES +// read the bytes still pays for it, and every queued landing is still applied, in order. +TEST_F(BufferTest, AWriteMapThatDiscardsWhatItMapsDoesNotReconcileAnAdoptedStoreAtMapTime) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kResidentSubDataMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_ARRAY_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + + // An earlier write is queued for its GPU-ordered landing... + const GLint firstInt = 55; + BufferSubData(GL_ARRAY_BUFFER, 0, sizeof(firstInt), &firstInt); + ASSERT_EQ(GetError(), GL_NO_ERROR); + ASSERT_EQ(mock.residentWrites.size(), 1u); + EXPECT_EQ(mock.readbackCalls, 0); + + // ...and the map that discards its range does not wait for it. + constexpr SizeT kFirst = 8; + constexpr SizeT kCount = 4; + const Range1D range{kFirst * sizeof(GLint), (kFirst + kCount) * sizeof(GLint)}; + auto* mapped = static_cast(bufferObject->AcquireMemoryRange( + range, BufferMappingAccessBit::Write | BufferMappingAccessBit::InvalidateRange)); + ASSERT_NE(mapped, nullptr); + EXPECT_EQ(mock.readbackCalls, 0); + for (SizeT i = 0; i < kCount; ++i) mapped[i] = 300 + static_cast(i); + bufferObject->ReleaseMemory(); + ASSERT_EQ(mock.residentWrites.size(), 2u); + + // The first read reconciles both landings, oldest first. + const auto* readBack = static_cast(bufferObject->AcquireMemory(false, true, false)); + EXPECT_EQ(mock.readbackCalls, 1); + EXPECT_EQ(readBack[0], firstInt); + for (SizeT i = 0; i < kCount; ++i) EXPECT_EQ(readBack[kFirst + i], 300 + static_cast(i)) << "int " << i; + + // The control: a map that keeps what it maps still reconciles before seeding. + BufferSubData(GL_ARRAY_BUFFER, 0, sizeof(firstInt), &firstInt); + ASSERT_EQ(GetError(), GL_NO_ERROR); + auto* seeded = static_cast( + bufferObject->AcquireMemoryRange(range, BufferMappingAccessBit::Read | BufferMappingAccessBit::Write)); + ASSERT_NE(seeded, nullptr); + EXPECT_EQ(mock.readbackCalls, 2); + for (SizeT i = 0; i < kCount; ++i) EXPECT_EQ(seeded[i], 300 + static_cast(i)) << "int " << i; + bufferObject->ReleaseMemory(); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +// Adoption releases the CPU shadow, and a persistent map that did not itself adopt +// (FLUSH_EXPLICIT is excluded from adoption) handed the application a pointer into that +// shadow which GL keeps valid while the buffer is drawn with - which is exactly when a +// storage binding asks for residency. So a mapped buffer keeps the shadow model. +TEST_F(BufferTest, AStorageBindingDoesNotAdoptTheStoreWhileTheApplicationHoldsAMapping) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + + GLuint buffer = 0; + GenBuffers(1, &buffer); + BindBuffer(GL_SHADER_STORAGE_BUFFER, buffer); + Vector initial(kAdoptedInts); + for (SizeT i = 0; i < kAdoptedInts; ++i) initial[i] = static_cast(i); + BufferData(GL_SHADER_STORAGE_BUFFER, static_cast(kAdoptedInts * sizeof(GLint)), initial.data(), + GL_DYNAMIC_DRAW); + ASSERT_EQ(GetError(), GL_NO_ERROR); + auto bufferObject = MG_State::pGLContext->GetBufferObject(buffer); + ASSERT_NE(bufferObject, nullptr); + const auto* shadowBase = bufferObject->MappedData(); + + constexpr SizeT kFirst = 2; + constexpr SizeT kCount = 4; + const Range1D range{kFirst * sizeof(GLint), (kFirst + kCount) * sizeof(GLint)}; + auto* mapped = static_cast(bufferObject->AcquireMemoryRange( + range, BufferMappingAccessBit::Write | BufferMappingAccessBit::Persistent | + BufferMappingAccessBit::FlushExplicit)); + ASSERT_NE(mapped, nullptr); + ASSERT_EQ(static_cast(mapped), static_cast(shadowBase + range.start)); + + EXPECT_FALSE(bufferObject->EnsureGpuResidentStorage()); + EXPECT_FALSE(bufferObject->IsBackendPersistentMapped()); + EXPECT_EQ(mock.acquireMapCalls, 0); + // The application's pointer is still the store's: it survived the binding. + EXPECT_EQ(static_cast(bufferObject->MappedData()), static_cast(shadowBase)); + for (SizeT i = 0; i < kCount; ++i) mapped[i] = 250 + static_cast(i); + bufferObject->FlushMemoryRange(0, kCount * sizeof(GLint)); + EXPECT_EQ(mock.flushCalls, 1); + const auto* shadowInts = reinterpret_cast(bufferObject->MappedData()); + for (SizeT i = 0; i < kCount; ++i) EXPECT_EQ(shadowInts[kFirst + i], 250 + static_cast(i)); + + // Unmapped, the next binding adopts as usual. + bufferObject->ReleaseMemory(); + EXPECT_TRUE(bufferObject->EnsureGpuResidentStorage()); + EXPECT_TRUE(bufferObject->IsBackendPersistentMapped()); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +// Respecifying a store hands any adoption back and replaces the bytes, so the staged +// bytes of a map that is still live have nowhere to land: copying a whole mapped range +// into storage that is released on the next line is pure waste. +TEST_F(BufferTest, RespecifyingAStoreWhileItIsMappedDoesNotLandTheStagedBytesIntoIt) { + ZeroCopyMockBackend mock; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kResidentSubDataMockOps); + GLuint buffer = 0; + auto bufferObject = MakeAdoptedBuffer(mock, GL_ARRAY_BUFFER, buffer); + ASSERT_NE(bufferObject, nullptr); + + auto* mapped = static_cast( + bufferObject->AcquireMemoryRange({0, kAdoptedInts * sizeof(GLint)}, BufferMappingAccessBit::Write)); + ASSERT_NE(mapped, nullptr); + for (SizeT i = 0; i < kAdoptedInts; ++i) mapped[i] = 1234; + + bufferObject->Respecify(kAdoptedInts * sizeof(GLint), nullptr); + EXPECT_TRUE(mock.residentWrites.empty()); + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.respecifyCalls, 1); + EXPECT_FALSE(bufferObject->IsMapped()); + EXPECT_FALSE(bufferObject->IsBackendPersistentMapped()); + EXPECT_FALSE(bufferObject->HasDefinedContent()); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} + +TEST_F(BufferTest, RespecifyingAShadowBackedStoreWhileItIsMappedPushesNoRangeDown) { + ZeroCopyMockBackend mock; + mock.provideMap = false; + g_zeroCopyMock = &mock; + ScopedBackendOps scopedOps(&kZeroCopyMockOps); + + GLuint buffer = 0; + GenBuffers(1, &buffer); + BindBuffer(GL_ARRAY_BUFFER, buffer); + Vector initial(kAdoptedInts, 7); + BufferData(GL_ARRAY_BUFFER, static_cast(kAdoptedInts * sizeof(GLint)), initial.data(), + GL_DYNAMIC_DRAW); + ASSERT_EQ(GetError(), GL_NO_ERROR); + auto bufferObject = MG_State::pGLContext->GetBufferObject(buffer); + ASSERT_NE(bufferObject, nullptr); + mock.flushCalls = 0; + mock.subDataCalls = 0; + mock.respecifyCalls = 0; + + auto* mapped = static_cast( + bufferObject->AcquireMemoryRange({0, kAdoptedInts * sizeof(GLint)}, BufferMappingAccessBit::Write)); + ASSERT_NE(mapped, nullptr); + for (SizeT i = 0; i < kAdoptedInts; ++i) mapped[i] = 1234; + + bufferObject->Respecify(kAdoptedInts * sizeof(GLint), nullptr); + EXPECT_EQ(mock.flushCalls, 0); + EXPECT_EQ(mock.subDataCalls, 0); + EXPECT_EQ(mock.respecifyCalls, 1); + EXPECT_FALSE(bufferObject->IsMapped()); + EXPECT_FALSE(bufferObject->HasDefinedContent()); + + DeleteBuffers(1, &buffer); + g_zeroCopyMock = nullptr; +} diff --git a/MobileGL/MG_Test/CMakeLists.txt b/MobileGL/MG_Test/CMakeLists.txt index a981087d..1fac40f3 100644 --- a/MobileGL/MG_Test/CMakeLists.txt +++ b/MobileGL/MG_Test/CMakeLists.txt @@ -64,6 +64,15 @@ set(LINK_LIBRARIES include(GoogleTest) gtest_discover_tests(SanityTest DISCOVERY_TIMEOUT 30 PROPERTIES LABELS unit) +if (MSVC) + # The GL headers declare gl* as dllimport on Windows, so any test that pulls + # GetProcAddress.cpp out of the static library references __imp_gl*, which only + # resolves when the in-library entry-point definitions are part of the link. + # Applies to every test executable below, the way the DirectVulkan and + # integration test targets already do it for themselves. + add_link_options(/WHOLEARCHIVE:MobileGL_s) +endif() + add_subdirectory(BackendLoader) add_subdirectory(Buffer) # The heap-address-is-not-an-identity invariant the backends' per-object memos diff --git a/MobileGL/MG_Test/Program/CMakeLists.txt b/MobileGL/MG_Test/Program/CMakeLists.txt index 712b19fc..95104cc4 100644 --- a/MobileGL/MG_Test/Program/CMakeLists.txt +++ b/MobileGL/MG_Test/Program/CMakeLists.txt @@ -23,6 +23,17 @@ target_link_libraries( ${LINK_LIBRARIES} ) +if (MSVC) + # This test compiles library sources of its own; pulling the whole static + # library in as well (the directory-wide MG_Test link option) would define + # them twice, so that option is dropped for this one target. + get_target_property(_program_util_link_options ProgramUtilTest LINK_OPTIONS) + if (_program_util_link_options) + list(REMOVE_ITEM _program_util_link_options /WHOLEARCHIVE:MobileGL_s) + set_target_properties(ProgramUtilTest PROPERTIES LINK_OPTIONS "${_program_util_link_options}") + endif() +endif() + add_executable( ProgramTest ProgramTest.cpp diff --git a/MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp b/MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp index 64b09d20..8696c237 100644 --- a/MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp +++ b/MobileGL/MG_Test/ShaderTranspiler/FlattenFloat64StorageBlockTest.cpp @@ -22,6 +22,8 @@ #include "Includes.h" #include "Init.h" #include +#include +#include #include @@ -38,6 +40,7 @@ namespace { constexpr Uint32 kOpTypeInt = 21; constexpr Uint32 kOpTypeFloat = 22; constexpr Uint32 kOpTypeArray = 28; + constexpr Uint32 kOpTypeRuntimeArray = 29; constexpr Uint32 kOpTypeStruct = 30; constexpr Uint32 kOpConstant = 43; constexpr Uint32 kDecorationArrayStride = 6; @@ -116,6 +119,18 @@ namespace { return {elementTypeId, length}; } + // The element type id of OpTypeRuntimeArray , or 0 when it is not one - which is + // what a BOUNDED flattened member (an OpTypeArray) answers too, so the two shapes can be told + // apart by the pair of helpers. + Uint32 RuntimeArrayElementOf(const Vector& spirv, Uint32 arrayId) { + Uint32 elementTypeId = 0; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode != kOpTypeRuntimeArray || wordCount < 3 || words[1] != arrayId) return; + elementTypeId = words[2]; + }); + return elementTypeId; + } + Bool IsUint32Type(const Vector& spirv, Uint32 typeId) { Bool isUint = false; ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { @@ -140,6 +155,61 @@ namespace { return text; } + // How many lines of a disassembly hold BOTH fragments - "OpIMul %uint" and "%uint_8", say - + // which is how the index arithmetic the pass emits is pinned without a host that could run it. + Uint32 CountLinesWith(const String& text, const String& first, const String& second) { + Uint32 count = 0; + SizeT lineStart = 0; + while (lineStart < text.size()) { + SizeT lineEnd = text.find('\n', lineStart); + if (lineEnd == String::npos) lineEnd = text.size(); + const String line = text.substr(lineStart, lineEnd - lineStart); + if (line.find(first) != String::npos && line.find(second) != String::npos) ++count; + lineStart = lineEnd + 1; + } + return count; + } + + // What every test of the open-ended shape asserts: the block collapsed to ONE member, which + // is a `uint[]` RUNTIME array of stride 4 rather than a bounded one, and nothing 64-bit is + // left for the demotion to find. Returns the disassembly for the arithmetic checks. + String ExpectOpenEndedWordArray(const Vector& output, const String& blockName) { + const String text = Disassemble(output); + const Uint32 structId = StructIdNamed(output, blockName); + EXPECT_NE(structId, 0u) << text; + if (structId == 0) return text; + const Vector members = MemberTypesOf(output, structId); + EXPECT_EQ(members.size(), 1u) << "the block should have collapsed to one member\n" << text; + if (members.size() != 1) return text; + EXPECT_EQ(MemberOffsetsOf(output, structId), (Vector{0})); + const Uint32 elementTypeId = RuntimeArrayElementOf(output, members[0]); + EXPECT_NE(elementTypeId, 0u) << "member 0 is not a runtime array\n" << text; + EXPECT_EQ(ArrayShapeOf(output, members[0]).first, 0u) + << "an open-ended block must not be given a bounded length\n" + << text; + EXPECT_TRUE(IsUint32Type(output, elementTypeId)) << text; + EXPECT_EQ(DecorationValueOf(output, members[0], kDecorationArrayStride), 4u) << text; + EXPECT_EQ(CountFloatTypesOfWidth(output, 64), 0u) << text; + return text; + } + + // The compute shape every failing KHR-Single-GL45.subgroups fp64 case binds: one runtime + // array of doubles, indexed by an invocation id, read whole-element. + String OpenEndedComputeSource(const String& elementType) { + return String(R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { )") + + elementType + R"( data[]; }; +void main() { + )" + elementType + + R"( value = data[gl_LocalInvocationID.x] * data[0]; + result[gl_GlobalInvocationID.x] = uint(value)" + + (elementType == "double" ? String{} : String(".x")) + R"(); +} +)"; + } + Vector CompileToSpirv(GLenum stage, const String& source) { using namespace MG_Util::ShaderTranspiler; ShaderAttrib shaderAttrib{.shaderType = stage, .sourceStr = source}; @@ -345,3 +415,679 @@ TEST_F(FlattenFloat64StorageBlockTest, TheDemotedPathIsUnchangedByTheCapabilityA EXPECT_EQ(explicitlyDemoted, defaulted); EXPECT_EQ(CountFloatTypesOfWidth(defaulted, 64), 0u) << Disassemble(defaulted); } + +// --------------------------------------------------------------------------- +// The open-ended shape: a block whose last member is a runtime array. Before this was accepted +// the pass declined it and the demotion re-derived ArrayStride 4 for the now-float element, so +// `double data[]` read the application's 8-byte-stride buffer as 32-bit words - every fp64 +// KHR-Single-GL45.subgroups case failed on exactly that. +// --------------------------------------------------------------------------- + +TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockOfDoublesBecomesAWordRuntimeArray) { + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource("double")); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + // Element i of the original array starts at word 2i, so the dynamic index is scaled by 2 ... + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 1u) << text; + // ... and the constant `data[0]` is the pair of words at 0 and 1, reached through the one + // member the block has left. + EXPECT_GE(CountLinesWith(text, "OpAccessChain %_ptr_StorageBuffer_uint", "%uint_0 %uint_0"), 1u) << text; +} + +TEST_F(FlattenFloat64StorageBlockTest, EachDoubleVectorWidthStepsByItsOwnStride) { + struct Shape { + const char* element; + // std430 strides: dvec2 16 bytes, dvec3 and dvec4 32 bytes - i.e. 4, 8 and 8 words. + const char* strideWords; + // The last component's word offset inside one element, and the first one past it. + const char* lastComponentWords; + const char* firstWordPastIt; + }; + const Shape shapes[] = {{"dvec2", "%uint_4", "%uint_2", "%uint_4"}, + {"dvec3", "%uint_8", "%uint_4", "%uint_6"}, + {"dvec4", "%uint_8", "%uint_6", "%uint_8"}}; + for (const Shape& shape : shapes) { + SCOPED_TRACE(shape.element); + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource(shape.element)); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", shape.strideWords), 1u) << text; + EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", shape.lastComponentWords), 1u) << text; + // A dvec3 is six words in a stride of eight: nothing may be read from the padding. + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", shape.firstWordPastIt), 0u) << text; + } +} + +TEST_F(FlattenFloat64StorageBlockTest, AFixedPrefixBeforeTheRuntimeArrayIsAddedToEveryIndex) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { + uvec4 head; + double data[]; +}; +void main() { + result[gl_GlobalInvocationID.x] = head.x + uint(data[gl_LocalInvocationID.x]); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Uint32 inputStructId = StructIdNamed(input, "Data"); + ASSERT_NE(inputStructId, 0u); + EXPECT_EQ(MemberOffsetsOf(input, inputStructId), (Vector{0, 16})) << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = ExpectOpenEndedWordArray(output, "Data"); + // The 16-byte prefix is 4 words: element i is at word 4 + 2i. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 1u) << text; + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_4"), 1u) << text; + // And the prefix member itself is still word 0. + EXPECT_GE(CountLinesWith(text, "OpAccessChain %_ptr_StorageBuffer_uint", "%uint_0 %uint_0"), 1u) << text; +} + +// OpArrayLength on the flattened member counts WORDS. GL's `.length()` is the number of whole +// elements the bound range holds past the array's offset, so the count has to be rebased and +// divided - in unsigned arithmetic, and clamped rather than wrapped when the range is shorter +// than the prefix. +namespace { + // A prefix, an open-ended array of doubles, and a `.length()` of it - the one shape whose + // rewrite is an instruction SPIRV-Cross has to spell rather than plain arithmetic. + constexpr const char* kOpenEndedLengthSource = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { + uvec4 head; + double data[]; +}; +void main() { + result[gl_GlobalInvocationID.x] = uint(data.length()) + head.y; +} +)"; +} // namespace + +TEST_F(FlattenFloat64StorageBlockTest, TheLengthOfAnOpenEndedBlockIsRewrittenToAnElementCount) { + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, kOpenEndedLengthSource); + ASSERT_FALSE(input.empty()); + // glslang asks for member 1's length and signs the answer. + EXPECT_EQ(CountLinesWith(Disassemble(input), "OpArrayLength %uint", " 1"), 1u) << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = ExpectOpenEndedWordArray(output, "Data"); + // Re-aimed at the one member left, ... + EXPECT_EQ(CountLinesWith(text, "OpArrayLength %uint", " 0"), 1u) << text; + EXPECT_EQ(CountLinesWith(text, "OpArrayLength %uint", " 1"), 0u) << text; + // ... rebased past the 4-word prefix, clamped at zero when the range does not reach it, ... + EXPECT_EQ(CountLinesWith(text, "OpISub %uint", "%uint_4"), 1u) << text; + EXPECT_EQ(CountLinesWith(text, "OpULessThan %bool", "%uint_4"), 1u) << text; + EXPECT_EQ(CountLinesWith(text, "OpSelect %uint", "%uint_0"), 1u) << text; + // ... and divided by the 2-word stride, with glslang's own sign conversion still downstream. + EXPECT_EQ(CountLinesWith(text, "OpUDiv %uint", "%uint_2"), 1u) << text; + EXPECT_EQ(CountLinesWith(text, "OpBitcast %int", ""), 1u) << text; +} + +TEST_F(FlattenFloat64StorageBlockTest, TheLengthOfABlockWithNoPrefixNeedsNoClamp) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { dvec2 data[]; }; +void main() { + result[gl_GlobalInvocationID.x] = uint(data.length()); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = ExpectOpenEndedWordArray(output, "Data"); + EXPECT_EQ(CountLinesWith(text, "OpArrayLength %uint", " 0"), 1u) << text; + // Nothing to subtract, so nothing to clamp: the word count over the 4-word stride is it. + EXPECT_EQ(CountLinesWith(text, "OpISub", ""), 0u) << text; + EXPECT_EQ(CountLinesWith(text, "OpSelect", ""), 0u) << text; + EXPECT_EQ(CountLinesWith(text, "OpUDiv %uint", "%uint_4"), 1u) << text; +} + +// The graphics shape of the same CTS group: a fragment stage reading a `readonly` block. The +// NonWritable the qualifier became is a promise about the whole block, and has to be on the one +// member the flattened block keeps. +TEST_F(FlattenFloat64StorageBlockTest, AReadOnlyOpenEndedBlockKeepsNonWritable) { + const String source = R"(#version 450 core +layout(binding = 4, std430) readonly buffer Buffer4 { dvec3 data[]; }; +layout(location = 0) out vec4 o_color; +void main() { + uint index = uint(gl_FragCoord.x); + o_color = vec4(float(data[index].z)); +} +)"; + const Vector input = CompileToSpirv(GL_FRAGMENT_SHADER, source); + ASSERT_FALSE(input.empty()); + EXPECT_EQ(CountLinesWith(Disassemble(input), "OpMemberDecorate %Buffer4 0 NonWritable", ""), 1u) + << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = ExpectOpenEndedWordArray(output, "Buffer4"); + EXPECT_EQ(CountLinesWith(text, "OpMemberDecorate %Buffer4 0 NonWritable", ""), 1u) << text; + // dvec3: stride 8 words, .z at +4. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_8"), 1u) << text; + EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", "%uint_4"), 1u) << text; +} + +// Writing through an open-ended block, which no CTS case does but any shader may: the store +// is decomposed into the same words the load would have read, so the bytes the application +// gets back are the ones GL says it wrote. +TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockIsWrittenThroughTheSameWords) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 1) buffer Data { double data[]; }; +void main() { + data[gl_LocalInvocationID.x] = double(gl_LocalInvocationID.y); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + // One dynamic index, scaled to the 2-word element ... + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 1u) << text; + // ... and the double left as exactly two word stores, nothing wider. + EXPECT_EQ(CountLinesWith(text, "OpStore", ""), 2u) << text; + EXPECT_EQ(CountLinesWith(text, "OpAccessChain %_ptr_StorageBuffer_uint", ""), 2u) << text; +} + +// The exact compute shader KHR-Single-GL45.subgroups.arithmetic.compute.subgroupmul_double +// generates, so the CTS shape is pinned as it is and not as a paraphrase of it. +TEST_F(FlattenFloat64StorageBlockTest, TheSubgroupMulDoubleComputeShaderIsFlattened) { + const String source = R"(#version 450 +#extension GL_KHR_shader_subgroup_arithmetic: enable +#extension GL_KHR_shader_subgroup_ballot: enable +layout (local_size_x = 16, local_size_y = 1, local_size_z = 1) in; +layout(binding = 0, std430) buffer Buffer0 +{ + uint result[]; +}; +layout(binding = 1, std430) buffer Buffer1 +{ + double data[]; +}; + +void main (void) +{ + uvec3 globalSize = gl_NumWorkGroups * gl_WorkGroupSize; + highp uint offset = globalSize.x * ((globalSize.y * gl_GlobalInvocationID.z) + gl_GlobalInvocationID.y) + gl_GlobalInvocationID.x; + uvec4 mask = subgroupBallot(true); + uint start = 0u, end = gl_SubgroupSize; + double ref = double(1); + uint tempResult = 0u; + for (uint index = start; index < end; index++) + { + if (subgroupBallotBitExtract(mask, index)) + { + ref = ref * data[index]; + } + } + tempResult = (abs(ref - subgroupMul(data[gl_SubgroupInvocationID])) < 0.00001) ? 0x1u : 0u; + if (1u == (gl_SubgroupInvocationID % 2u)) + { + mask = subgroupBallot(true); + ref = double(1); + for (uint index = start; index < end; index++) + { + if (subgroupBallotBitExtract(mask, index)) + { + ref = ref * data[index]; + } + } + tempResult |= (abs(ref - subgroupMul(data[gl_SubgroupInvocationID])) < 0.00001) ? 0x2u : 0u; + } + else + { + tempResult |= 0x2u; + } + result[offset] = tempResult; +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Buffer1"); + // Four reads of the array, each scaled to the 2-word element. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2"), 4u) << text; + // The result block holds no double and is not the pass's business. + const Uint32 resultStructId = StructIdNamed(output, "Buffer0"); + ASSERT_NE(resultStructId, 0u) << text; + const Vector resultMembers = MemberTypesOf(output, resultStructId); + ASSERT_EQ(resultMembers.size(), 1u); + EXPECT_TRUE(IsUint32Type(output, RuntimeArrayElementOf(output, resultMembers[0]))) << text; +} + +// A runtime array whose element is a MATRIX. The member's own MatrixStride and RowMajor +// decorations describe those elements, so a row-major one has to be declined - its columns are +// not contiguous, and addressing it in column order against a row-major buffer would be silently +// wrong bytes rather than a refusal. The column-major twin must flatten, stepping by the +// element's stride and then by the column's. +TEST_F(FlattenFloat64StorageBlockTest, ARowMajorMatrixRuntimeArrayIsLeftToTheDemotion) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1, row_major) buffer Data { dmat4 data[]; }; +void main() { + result[gl_GlobalInvocationID.x] = uint(data[gl_LocalInvocationID.x][1][2]); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + // The premise: glslang really did mark the member row-major. + EXPECT_EQ(CountLinesWith(Disassemble(input), "OpMemberDecorate %Data 0 RowMajor", ""), 1u) + << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = Disassemble(output); + const Uint32 structId = StructIdNamed(output, "Data"); + ASSERT_NE(structId, 0u) << text; + const Vector members = MemberTypesOf(output, structId); + ASSERT_EQ(members.size(), 1u) << text; + // Still a runtime array of matrices - narrowed to fp32 by the demotion, not re-addressed. + EXPECT_NE(DecorationValueOf(output, members[0], kDecorationArrayStride), 4u) + << "a row-major matrix element must not have been flattened into words\n" + << text; + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", ""), 0u) + << "nothing should have been re-addressed\n" + << text; +} + +TEST_F(FlattenFloat64StorageBlockTest, AColumnMajorMatrixRuntimeArrayStepsByItsColumnStride) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { dmat2x4 data[]; }; +void main() { + dvec4 column = data[gl_LocalInvocationID.x][1]; + result[gl_GlobalInvocationID.x] = uint(column.w); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + // dmat2x4: two columns of dvec4, column stride 32 bytes, so one element is 64 bytes - + // 16 words - and column 1 starts 8 words into it. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_16"), 1u) << text; + // Exactly one +8: the column's own offset inside the element. A second would mean a word + // past the column was being addressed off that same base. + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_8"), 1u) << text; + // All eight words of that column are read - the last of its four doubles ends at +7 ... + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_7"), 1u) << text; + // ... and the column that was not asked for is not touched: nothing is read at +9 or past. + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_9"), 0u) << text; + EXPECT_EQ(CountLinesWith(text, "OpIAdd %uint", "%uint_10"), 0u) << text; +} + +// A runtime array whose element is a STRUCT: the same walk, and the same decline test, as a +// bounded array of them - a shape no other open-ended case reaches. +TEST_F(FlattenFloat64StorageBlockTest, AStructRuntimeArrayStepsByItsElementStride) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +struct Pair { double a; float b; }; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { Pair data[]; }; +void main() { + result[gl_GlobalInvocationID.x] = uint(data[gl_LocalInvocationID.x].a) + + uint(data[gl_LocalInvocationID.x].b); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + // std430 rounds `{ double a; float b; }` up to its 8-byte alignment: 16 bytes, 4 words, + // with `b` two words in. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_4"), 2u) << text; + EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", "%uint_2"), 1u) << text; +} + +// The leaf cap bounds ONE load or store, not a member's size: a block whose element is far too +// big to expand whole is still flattened while every access to it names a scalar. Declining it +// would leave the application's 8-byte-stride doubles to the demotion's re-derived stride 4 - +// the exact defect the open-ended shape exists to avoid. +TEST_F(FlattenFloat64StorageBlockTest, AHugeRuntimeArrayElementIsStillFlattenedWhenAccessesAreSmall) { + const String source = R"(#version 430 core +layout(local_size_x = 16) in; +struct Big { dvec4 v[300]; }; +layout(std430, binding = 0) buffer Sink { uint result[]; }; +layout(std430, binding = 1) buffer Data { Big data[]; }; +void main() { + result[gl_GlobalInvocationID.x] = uint(data[gl_LocalInvocationID.x].v[3].y); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + + const String text = ExpectOpenEndedWordArray(output, "Data"); + // 300 dvec4 of 32 bytes each: 9600 bytes, 2400 words per element - 1200 scalars, well past + // the per-access cap that a whole-element load would have to respect and this never does. + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", "%uint_2400"), 1u) << text; + // v[3].y is 3 * 8 + 2 = 26 words into the element. + EXPECT_GE(CountLinesWith(text, "OpIAdd %uint", "%uint_26"), 1u) << text; +} + +// The flatten preserves a byte layout ACROSS a narrowing; where the backend consumes 64-bit +// floats itself there is nothing to preserve, and the open-ended block has to keep its runtime +// array of doubles exactly as the driver would lay it out. +TEST_F(FlattenFloat64StorageBlockTest, TheNativePathLeavesAnOpenEndedBlockAndItsDoublesAlone) { + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource("double")); + ASSERT_FALSE(input.empty()); + + Vector output; + ASSERT_TRUE(ShaderCompiler::SanitizeAndOptimizeBinary(input, output, true, true, true)); + ASSERT_FALSE(output.empty()); + + const String text = Disassemble(output); + const Uint32 structId = StructIdNamed(output, "Data"); + ASSERT_NE(structId, 0u) << text; + const Vector members = MemberTypesOf(output, structId); + ASSERT_EQ(members.size(), 1u) << text; + EXPECT_NE(RuntimeArrayElementOf(output, members[0]), 0u) + << "the member should still be a runtime array\n" + << text; + EXPECT_EQ(DecorationValueOf(output, members[0], kDecorationArrayStride), 8u) + << "the array must keep the 8-byte stride the application bound\n" + << text; + EXPECT_GT(CountFloatTypesOfWidth(output, 64), 0u) + << "nothing narrows here, so the doubles must survive\n" + << text; +} + +// The other backend prints the flattened module through SPIRV-Cross: an open-ended `uint[]` +// member has to come out as ESSL that names no 64-bit type. The `.length()` shape is here too, +// because the OpArrayLength the rewrite re-issues is the one instruction in it whose ESSL +// spelling is not plain arithmetic - if that backend ever refused it on the flattened member, +// a DirectGLES shader asking an fp64 buffer its length would fail at link and nowhere else. +namespace { + String TranspileToEssl(const Vector& spirv) { + using namespace MG_Util::ShaderTranspiler; + SpvcSession session(spirv, SessionUsageBit::Transpile); + spvc_compiler_options options; + EXPECT_EQ(session.CreateOptions(&options), SPVC_SUCCESS); + spvc_compiler_options_set_uint(options, SPVC_COMPILER_OPTION_GLSL_VERSION, 320); + spvc_compiler_options_set_bool(options, SPVC_COMPILER_OPTION_GLSL_ES, SPVC_TRUE); + spvc_compiler_options_set_bool(options, SPVC_COMPILER_OPTION_GLSL_VULKAN_SEMANTICS, SPVC_FALSE); + EXPECT_EQ(session.SetOptions(options), SPVC_SUCCESS); + + auto essl = ShaderCompiler::DecompileShader(session); + EXPECT_TRUE(essl) << (essl ? String{} : essl.error().log); + return essl ? *essl : String{}; + } +} // namespace + +TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockCanBeEmittedAsEssl) { + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, OpenEndedComputeSource("dvec4")); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + ExpectOpenEndedWordArray(output, "Data"); + + const String essl = TranspileToEssl(output); + ASSERT_FALSE(essl.empty()); + EXPECT_EQ(essl.find("double"), String::npos) << essl; + EXPECT_EQ(essl.find("dvec"), String::npos) << essl; + EXPECT_NE(essl.find("uint"), String::npos) << essl; +} + +TEST_F(FlattenFloat64StorageBlockTest, TheRewrittenLengthCanBeEmittedAsEssl) { + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, kOpenEndedLengthSource); + ASSERT_FALSE(input.empty()); + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + ExpectOpenEndedWordArray(output, "Data"); + + const String essl = TranspileToEssl(output); + ASSERT_FALSE(essl.empty()); + EXPECT_EQ(essl.find("double"), String::npos) << essl; + EXPECT_EQ(essl.find("dvec"), String::npos) << essl; + // The length survived as a length - it was not folded away or dropped on the floor. + EXPECT_NE(essl.find(".length()"), String::npos) << essl; +} + +// --------------------------------------------------------------------------- +// The gate from the other side: a runtime array anywhere but the block's own last member is a +// shape GLSL cannot spell and this pass does not describe. SPIR-V can spell it, so both are +// hand-written, and both are invalid Vulkan SPIR-V - the chain runs without its validator here, +// which is also why neither can be a validation-failure count. +// --------------------------------------------------------------------------- + +namespace { + // `buffer Odd { double data[]; uint tail; }`, the runtime array FIRST. + const char* kRuntimeArrayNotLastAsm = R"( + OpCapability Shader + OpCapability Float64 + OpMemoryModel Logical GLSL450 + OpEntryPoint GLCompute %main "main" + OpExecutionMode %main LocalSize 1 1 1 + OpName %Odd "Odd" + OpName %var "" + OpDecorate %_runtimearr_double ArrayStride 8 + OpDecorate %Odd Block + OpMemberDecorate %Odd 0 Offset 0 + OpMemberDecorate %Odd 1 Offset 8 + OpDecorate %var Binding 0 + OpDecorate %var DescriptorSet 0 + %void = OpTypeVoid + %3 = OpTypeFunction %void + %uint = OpTypeInt 32 0 + %int = OpTypeInt 32 1 + %int_0 = OpConstant %int 0 + %int_1 = OpConstant %int 1 + %double = OpTypeFloat 64 + %double_2 = OpConstant %double 2 +%_runtimearr_double = OpTypeRuntimeArray %double + %Odd = OpTypeStruct %_runtimearr_double %uint +%_ptr_StorageBuffer_Odd = OpTypePointer StorageBuffer %Odd + %var = OpVariable %_ptr_StorageBuffer_Odd StorageBuffer +%_ptr_StorageBuffer_double = OpTypePointer StorageBuffer %double + %main = OpFunction %void None %3 + %5 = OpLabel + %6 = OpAccessChain %_ptr_StorageBuffer_double %var %int_0 %int_1 + OpStore %6 %double_2 + OpReturn + OpFunctionEnd +)"; + + // `struct Inner { double data[]; }; buffer Outer { uint head; Inner inner; }`: the runtime + // array IS last, but of a member rather than of the block. + const char* kRuntimeArrayNestedAsm = R"( + OpCapability Shader + OpCapability Float64 + OpMemoryModel Logical GLSL450 + OpEntryPoint GLCompute %main "main" + OpExecutionMode %main LocalSize 1 1 1 + OpName %Outer "Outer" + OpName %Inner "Inner" + OpName %var "" + OpDecorate %_runtimearr_double ArrayStride 8 + OpMemberDecorate %Inner 0 Offset 0 + OpDecorate %Outer Block + OpMemberDecorate %Outer 0 Offset 0 + OpMemberDecorate %Outer 1 Offset 8 + OpDecorate %var Binding 0 + OpDecorate %var DescriptorSet 0 + %void = OpTypeVoid + %3 = OpTypeFunction %void + %uint = OpTypeInt 32 0 + %int = OpTypeInt 32 1 + %int_0 = OpConstant %int 0 + %int_1 = OpConstant %int 1 + %double = OpTypeFloat 64 + %double_2 = OpConstant %double 2 +%_runtimearr_double = OpTypeRuntimeArray %double + %Inner = OpTypeStruct %_runtimearr_double + %Outer = OpTypeStruct %uint %Inner +%_ptr_StorageBuffer_Outer = OpTypePointer StorageBuffer %Outer + %var = OpVariable %_ptr_StorageBuffer_Outer StorageBuffer +%_ptr_StorageBuffer_double = OpTypePointer StorageBuffer %double + %main = OpFunction %void None %3 + %5 = OpLabel + %6 = OpAccessChain %_ptr_StorageBuffer_double %var %int_1 %int_0 %int_1 + OpStore %6 %double_2 + OpReturn + OpFunctionEnd +)"; + + Vector AssembleUnchecked(const char* asmText) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1); + Vector module; + EXPECT_TRUE(tools.Assemble(asmText, &module)); + return module; + } +} // namespace + +TEST_F(FlattenFloat64StorageBlockTest, ARuntimeArrayThatIsNotTheBlocksLastMemberIsLeftToTheDemotion) { + struct Shape { + const char* asmText; + const char* blockName; + }; + const Shape shapes[] = {{kRuntimeArrayNotLastAsm, "Odd"}, {kRuntimeArrayNestedAsm, "Outer"}}; + for (const Shape& shape : shapes) { + SCOPED_TRACE(shape.blockName); + const Vector input = AssembleUnchecked(shape.asmText); + ASSERT_FALSE(input.empty()); + + Vector output; + ASSERT_TRUE(ShaderCompiler::SanitizeAndOptimizeBinary(input, output, false, false)); + ASSERT_FALSE(output.empty()); + const String text = Disassemble(output); + + // Declined: both members are still there, and the demotion narrowed them the old way. + const Uint32 structId = StructIdNamed(output, shape.blockName); + ASSERT_NE(structId, 0u) << text; + EXPECT_EQ(MemberTypesOf(output, structId).size(), 2u) << text; + EXPECT_EQ(CountFloatTypesOfWidth(output, 64), 0u) << text; + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", ""), 0u) + << "nothing should have been re-addressed\n" + << text; + } +} + +// --------------------------------------------------------------------------- +// The front end declares types in first-use order, so a block that is the first thing the +// shader touches is declared before the module's `uint` - and the flattened member is an array +// OF `uint`. For an OPEN-ENDED block the pass moves that operand-less type up in front of the +// block rather than declining, so that where a buffer of doubles stands in the shader does not +// decide whether its bytes survive. A BOUNDED block in the same position keeps the decline it +// has always had: widening that is a change to a path this fix does not need, and the pair below +// pins both halves. +// --------------------------------------------------------------------------- + +namespace { + // The position of 's declaration in instruction order, or 0 when it has none. + Uint32 DeclarationIndexOf(const Vector& spirv, Uint32 id) { + Uint32 index = 0; + Uint32 found = 0; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + ++index; + if (found != 0 || wordCount < 2) return; + // Every OpType* has its result id in word 1; that is all this is asked about. + if (opcode >= kOpTypeInt && opcode <= kOpTypeStruct && words[1] == id) found = index; + }); + return found; + } + + Uint32 Uint32TypeIdOf(const Vector& spirv) { + Uint32 typeId = 0; + ForEachInstruction(spirv, [&](Uint32 opcode, const Uint32* words, Uint32 wordCount) { + if (opcode == kOpTypeInt && wordCount >= 4 && words[2] == 32u && words[3] == 0u) typeId = words[1]; + }); + return typeId; + } +} // namespace + +TEST_F(FlattenFloat64StorageBlockTest, AnOpenEndedBlockDeclaredBeforeTheModulesUintIsStillFlattened) { + // The block is the first thing main touches, and nothing before it needs a uint - not even + // an array length, which is a uint constant and would declare one. + const String source = R"(#version 430 core +layout(local_size_x = 1) in; +layout(std430, binding = 0) buffer Data { double data[]; }; +layout(std430, binding = 1) buffer Sink { float result[]; }; +void main() { + result[0] = float(data[0] + data[1]); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Uint32 inputStructId = StructIdNamed(input, "Data"); + ASSERT_NE(inputStructId, 0u); + const Uint32 inputUintId = Uint32TypeIdOf(input); + // The premise: the module's uint really is declared after the block (or not at all). + ASSERT_TRUE(inputUintId == 0 || + DeclarationIndexOf(input, inputUintId) > DeclarationIndexOf(input, inputStructId)) + << "this shader was meant to declare the block before any uint\n" + << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = ExpectOpenEndedWordArray(output, "Data"); + const Uint32 structId = StructIdNamed(output, "Data"); + ASSERT_NE(structId, 0u) << text; + const Vector members = MemberTypesOf(output, structId); + ASSERT_EQ(members.size(), 1u) << text; + // And the uint now stands in front of the block it is an element of. + EXPECT_LT(DeclarationIndexOf(output, RuntimeArrayElementOf(output, members[0])), + DeclarationIndexOf(output, structId)) + << text; +} + +TEST_F(FlattenFloat64StorageBlockTest, ABoundedBlockDeclaredBeforeTheModulesUintIsLeftToTheDemotion) { + // The same position, a bounded block: this is the shape that has always been declined, and + // it stays declined - its members and the demotion's own repacking come through untouched. + const String source = R"(#version 430 core +layout(local_size_x = 1) in; +layout(std430, binding = 0) buffer Wide { + double data0; + dvec2 data1; +} g_wide; +layout(std430, binding = 1) buffer Sink { float result[]; }; +void main() { + double sum = g_wide.data0 + g_wide.data1.y; + result[0] = float(sum); +} +)"; + const Vector input = CompileToSpirv(GL_COMPUTE_SHADER, source); + ASSERT_FALSE(input.empty()); + const Uint32 inputStructId = StructIdNamed(input, "Wide"); + ASSERT_NE(inputStructId, 0u); + const Uint32 inputUintId = Uint32TypeIdOf(input); + ASSERT_TRUE(inputUintId == 0 || + DeclarationIndexOf(input, inputUintId) > DeclarationIndexOf(input, inputStructId)) + << "this shader was meant to declare the block before any uint\n" + << Disassemble(input); + + const Vector output = Sanitize(input); + ASSERT_FALSE(output.empty()); + const String text = Disassemble(output); + const Uint32 structId = StructIdNamed(output, "Wide"); + ASSERT_NE(structId, 0u) << text; + EXPECT_EQ(MemberTypesOf(output, structId).size(), 2u) + << "a bounded block in this position must keep the decline it shipped with\n" + << text; + EXPECT_EQ(CountLinesWith(text, "OpIMul %uint", ""), 0u) + << "nothing should have been re-addressed\n" + << text; +} diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp index 66a62037..7f920ccf 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.cpp @@ -84,8 +84,18 @@ namespace MobileGL { struct BlockPlan { Instruction* structType = nullptr; uint32_t storageClass = 0; + // A bounded block's length in words. For an open-ended block - one whose + // last member is a runtime array - the FIXED PREFIX in words, i.e. the + // runtime array's own offset, which is where its element 0 starts. uint32_t wordCount = 0; + bool openEnded = false; + // The original runtime array's stride in words; what one element of it + // steps by, and what its word count divides by to become a length. + uint32_t tailStrideWords = 0; std::vector chains; + // The OpArrayLength users of an open-ended block's variables, which count + // WORDS once the member is a `uint[]` and so have to be rewritten too. + std::vector arrayLengths; }; bool IsDoubleType(const Instruction* type) { @@ -178,8 +188,9 @@ namespace MobileGL { } // Byte size of a type as it is laid out INSIDE a block, or 0 when this pass - // cannot describe it (a runtime array, a width it does not carry, a matrix with - // no stride or a row-major one). + // cannot describe it (a runtime array - the one place a block may have one is + // its last member, which MeasureBlock handles above this - a width it does not + // carry, a matrix with no stride or a row-major one). uint32_t LaidOutByteSize(IRContext* context, const TypeCursor& cursor) { const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId); if (type == nullptr) return 0; @@ -231,8 +242,17 @@ namespace MobileGL { } // Whether this type decomposes into scalars the rewrite can move one word at a - // time, counting them so a whole-aggregate access can be refused before it is - // expanded. + // time. |leafCount| counts them, so a whole-aggregate access can be refused + // before it is expanded; passing NULL asks the SHAPE question alone - is this + // type addressable at all - and then identical array elements and vector + // components are walked once instead of once each, because the answer cannot + // differ between them and the walk of a big one would not be free. + // + // The two questions are separate because only a LOAD or a STORE expands into + // leaves, and the cap bounds one of those. How large a runtime array's element + // is says nothing about how many scalars a single access to it moves, so + // MeasureBlock asks for the shape and BuildPlans applies the cap where it + // belongs - per chain, to the type that chain actually names. bool CanDecompose(IRContext* context, const TypeCursor& cursor, uint32_t* leafCount) { const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId); if (type == nullptr) return false; @@ -240,12 +260,15 @@ namespace MobileGL { case spv::Op::OpTypeInt: case spv::Op::OpTypeFloat: if (ScalarByteSize(type) == 0) return false; + if (leafCount == nullptr) return true; ++*leafCount; return *leafCount <= kMaxLeavesPerAccess; case spv::Op::OpTypeVector: { TypeCursor component; component.typeId = type->GetSingleWordInOperand(0); - for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) { + const uint32_t repeats = + leafCount == nullptr ? 1u : type->GetSingleWordInOperand(1); + for (uint32_t i = 0; i < repeats; ++i) { if (!CanDecompose(context, component, leafCount)) return false; } return true; @@ -257,7 +280,9 @@ namespace MobileGL { } TypeCursor column; column.typeId = type->GetSingleWordInOperand(0); - for (uint32_t i = 0; i < type->GetSingleWordInOperand(1); ++i) { + const uint32_t repeats = + leafCount == nullptr ? 1u : type->GetSingleWordInOperand(1); + for (uint32_t i = 0; i < repeats; ++i) { if (!CanDecompose(context, column, leafCount)) return false; } return true; @@ -273,10 +298,12 @@ namespace MobileGL { context->get_constant_mgr()->FindDeclaredConstant(type->GetSingleWordInOperand(1)); if (length == nullptr || length->AsIntConstant() == nullptr) return false; const uint32_t count = length->AsIntConstant()->GetU32BitValue(); - if (count == 0 || count > kMaxLeavesPerAccess) return false; + if (count == 0) return false; + if (leafCount != nullptr && count > kMaxLeavesPerAccess) return false; TypeCursor element = cursor; element.typeId = type->GetSingleWordInOperand(0); - for (uint32_t i = 0; i < count; ++i) { + const uint32_t repeats = leafCount == nullptr ? 1u : count; + for (uint32_t i = 0; i < repeats; ++i) { if (!CanDecompose(context, element, leafCount)) return false; } return true; @@ -300,6 +327,63 @@ namespace MobileGL { } } + // Measures the block struct itself. A bounded block reports its laid-out byte + // size; a block whose LAST member is a runtime array - the only place GLSL lets + // one stand, and the only place SPIR-V lets a Block have one - reports the byte + // offset that array starts at and says so through |openEnded|, with the array's + // stride alongside. A runtime array anywhere else, one without a stride the + // words can step by, or one whose element the rewrite could not take apart is a + // shape this pass does not describe, and so is a bounded block it cannot size. + bool MeasureBlock(IRContext* context, const Instruction* structType, uint32_t* bytes, + bool* openEnded, uint32_t* tailStrideBytes) { + *bytes = 0; + *openEnded = false; + *tailStrideBytes = 0; + const uint32_t structId = structType->result_id(); + const uint32_t memberCount = structType->NumInOperands(); + uint64_t end = 0; + for (uint32_t member = 0; member < memberCount; ++member) { + uint32_t offset = 0; + if (!TryGetMemberDecorationLiteral(context, structId, member, spv::Decoration::Offset, + &offset)) { + return false; + } + const TypeCursor cursor = MemberCursor(context, structType, member); + const Instruction* type = context->get_def_use_mgr()->GetDef(cursor.typeId); + if (type == nullptr) return false; + if (type->opcode() == spv::Op::OpTypeRuntimeArray) { + if (member + 1 != memberCount) return false; + uint32_t stride = 0; + if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride, + &stride) || + stride == 0 || stride % kWordBytes != 0) { + return false; + } + // The member's own matrix decorations describe the array's ELEMENTS, + // exactly as they do for a bounded array of matrices. Only the shape + // is asked for: how big one element is decides nothing about how + // many scalars one access moves, and a leaf cap here would decline a + // block over a member the shader may never read whole. + TypeCursor element = cursor; + element.typeId = type->GetSingleWordInOperand(0); + if (!CanDecompose(context, element, nullptr)) return false; + // Element 0 has to start past every fixed member, or the words the + // prefix owns and the words the array owns would overlap. + if (offset < end) return false; + end = offset; + *openEnded = true; + *tailStrideBytes = stride; + break; + } + const uint32_t size = LaidOutByteSize(context, cursor); + if (size == 0) return false; + end = std::max(end, static_cast(offset) + size); + } + if (end > kMaxBlockBytes) return false; + *bytes = static_cast(end); + return true; + } + bool TypeContainsFloat64(IRContext* context, uint32_t typeId, std::unordered_set& visiting) { const Instruction* type = context->get_def_use_mgr()->GetDef(typeId); @@ -366,6 +450,9 @@ namespace MobileGL { switch (type->opcode()) { case spv::Op::OpTypeArray: + // A runtime array steps exactly like a bounded one; only its end is + // unknown, and a chain never needs that. + case spv::Op::OpTypeRuntimeArray: if (!TryGetDecorationLiteral(context, cursor.typeId, spv::Decoration::ArrayStride, &stride)) { return false; @@ -611,6 +698,39 @@ namespace MobileGL { } } + // Replaces an OpArrayLength of an open-ended block with the element count + // of the ORIGINAL runtime array. The instruction now counts the words of + // the flattened `uint[]`, so the length is `(words - prefix) / stride`, in + // unsigned arithmetic and clamped at zero when the bound range does not + // even reach the array's offset - a wrapped subtraction would otherwise + // report a few billion elements. The division floors, which is what GL + // defines `.length()` as for a range that is not a whole number of + // elements. A fresh OpArrayLength is issued rather than the old one re-aimed, + // so the uses being redirected are never the ones the arithmetic just made. + void RewriteArrayLength(Instruction* arrayLength, uint32_t prefixWords, uint32_t strideWords) { + InstructionBuilder builder(m_context, arrayLength, kPreservedAnalyses); + const uint32_t variableId = arrayLength->GetSingleWordInOperand(0); + const uint32_t wordsId = m_context->TakeNextId(); + builder.AddInstruction(MakeUnique( + m_context, spv::Op::OpArrayLength, m_uintTypeId, wordsId, + std::initializer_list{{SPV_OPERAND_TYPE_ID, {variableId}}, + {SPV_OPERAND_TYPE_LITERAL_INTEGER, {0u}}})); + uint32_t count = wordsId; + if (prefixWords != 0) { + const uint32_t prefixId = UintConstant(prefixWords); + const uint32_t past = Binary(builder, spv::Op::OpISub, m_uintTypeId, count, prefixId); + const uint32_t tooShort = + Binary(builder, spv::Op::OpULessThan, m_boolTypeId, count, prefixId); + count = Select(builder, tooShort, UintConstant(0), past); + } + if (strideWords != 1) { + count = Binary(builder, spv::Op::OpUDiv, m_uintTypeId, count, + UintConstant(strideWords)); + } + m_context->ReplaceAllUsesWith(arrayLength->result_id(), count); + m_context->KillInst(arrayLength); + } + private: uint32_t ComponentWords(uint32_t componentTypeId) { return ScalarByteSize(m_context->get_def_use_mgr()->GetDef(componentTypeId)) / @@ -788,38 +908,72 @@ namespace MobileGL { return false; } - // A fresh `uint[length]` with ArrayStride 4, spliced in immediately BEFORE the - // block that will name it - SPIR-V has no forward references between types, so - // appending it at the end of the section would make the module invalid. A - // duplicate OpTypeArray is legal (SPIR-V 2.8 exempts aggregates from the - // uniqueness rule, and so does spirv-val), so no search for an existing one is - // needed; the LENGTH CONSTANT is not exempt, and if the module already declares - // it after the block there is nowhere legal to put the array - the block is then - // declined and keeps today's behaviour. Returns 0 for that, and for a uint type - // that is itself declared too late. + // A fresh `uint[length]` with ArrayStride 4 - or, for an open-ended block, a + // `uint[]` runtime array with the same stride and no length at all - spliced in + // immediately BEFORE the block that will name it: SPIR-V has no forward + // references between types, so appending it at the end of the section would make + // the module invalid. A duplicate OpTypeArray or OpTypeRuntimeArray is legal + // (SPIR-V 2.8 exempts aggregates from the uniqueness rule, and so does + // spirv-val), so no search for an existing one is needed; the LENGTH CONSTANT is + // not exempt, and if the module already declares it after the block there is + // nowhere legal to put the array - the block is then declined and keeps today's + // behaviour. Returns 0 for that; an open-ended block has no length constant to + // place, so that reason cannot reach it. + // + // The `uint` element type is a different matter, and only for an OPEN-ENDED + // block. The front end declares types in first-use order, so a block that is the + // first thing a shader touches sits BEFORE the module's `uint` (or the module has + // none, and the one the pass asked for was appended at the end). Declining there + // would send exactly the buffers this rewrite exists for back to the demotion on + // nothing but where they stand in the source. OpTypeInt has no operands, and + // nothing that names it can precede where it was, so moving it up in front of the + // block is always legal. A BOUNDED block keeps declining instead: that is what it + // has always done, and widening it is a change to a path this one does not need. + // + // NOTHING IS WRITTEN until every reason to decline has been ruled out, so a block + // this returns 0 for leaves the module as it found it - which is what lets + // Process() truthfully report SuccessWithoutChange for a module of only those. uint32_t CreateWordArrayTypeBefore(IRContext* context, Instruction* structType, - uint32_t uintTypeId, uint32_t length) { - if (!DeclaredBefore(context, uintTypeId, structType->result_id())) return 0; + uint32_t uintTypeId, uint32_t length, bool openEnded) { + Instruction* uintType = context->get_def_use_mgr()->GetDef(uintTypeId); + if (uintType == nullptr || uintType->opcode() != spv::Op::OpTypeInt) return 0; + const bool hoistUint = !DeclaredBefore(context, uintTypeId, structType->result_id()); + if (hoistUint && !openEnded) return 0; - auto* constantMgr = context->get_constant_mgr(); - const spvtools::opt::analysis::Type* uintType = context->get_type_mgr()->GetType(uintTypeId); - if (uintType == nullptr) return 0; - const spvtools::opt::analysis::Constant* lengthConstant = - constantMgr->GetConstant(uintType, {length}); - if (lengthConstant == nullptr) return 0; + uint32_t lengthConstantId = 0; + if (!openEnded) { + auto* constantMgr = context->get_constant_mgr(); + const spvtools::opt::analysis::Type* uintDescriptor = + context->get_type_mgr()->GetType(uintTypeId); + if (uintDescriptor == nullptr) return 0; + const spvtools::opt::analysis::Constant* lengthConstant = + constantMgr->GetConstant(uintDescriptor, {length}); + if (lengthConstant == nullptr) return 0; - Module::inst_iterator position = PositionOf(context, structType); - if (position == context->types_values_end()) return 0; - Instruction* lengthInst = constantMgr->GetDefiningInstruction(lengthConstant, 0, &position); - if (lengthInst == nullptr) return 0; - if (!DeclaredBefore(context, lengthInst->result_id(), structType->result_id())) return 0; + Module::inst_iterator position = PositionOf(context, structType); + if (position == context->types_values_end()) return 0; + // Created in front of the block when it is not there yet, so the only way + // this declines is a constant the module already declares after it. + Instruction* lengthInst = + constantMgr->GetDefiningInstruction(lengthConstant, 0, &position); + if (lengthInst == nullptr) return 0; + if (!DeclaredBefore(context, lengthInst->result_id(), structType->result_id())) return 0; + lengthConstantId = lengthInst->result_id(); + } const uint32_t arrayTypeId = context->TakeNextId(); if (arrayTypeId == 0) return 0; - auto arrayType = MakeUnique( - context, spv::Op::OpTypeArray, 0, arrayTypeId, - std::initializer_list{{SPV_OPERAND_TYPE_ID, {uintTypeId}}, - {SPV_OPERAND_TYPE_ID, {lengthInst->result_id()}}}); + + if (hoistUint) uintType->InsertBefore(structType); + std::unique_ptr arrayType = + openEnded + ? MakeUnique( + context, spv::Op::OpTypeRuntimeArray, 0, arrayTypeId, + std::initializer_list{{SPV_OPERAND_TYPE_ID, {uintTypeId}}}) + : MakeUnique( + context, spv::Op::OpTypeArray, 0, arrayTypeId, + std::initializer_list{{SPV_OPERAND_TYPE_ID, {uintTypeId}}, + {SPV_OPERAND_TYPE_ID, {lengthConstantId}}}); Instruction* inserted = structType->InsertBefore(std::move(arrayType)); context->AnalyzeDefUse(inserted); context->get_decoration_mgr()->AddDecorationVal( @@ -948,8 +1102,14 @@ namespace MobileGL { Instruction* structType = defUseMgr->GetDef(structId); TypeCursor blockCursor; blockCursor.typeId = structId; - const uint32_t blockBytes = LaidOutByteSize(context, blockCursor); - if (blockBytes == 0 || blockBytes % kWordBytes != 0) { + uint32_t blockBytes = 0; + bool openEnded = false; + uint32_t tailStrideBytes = 0; + // An open-ended block whose runtime array is its only member measures a + // prefix of 0 bytes and is perfectly describable; only a BOUNDED block of + // no bytes is not, and MeasureBlock already refuses to size one of those. + if (!MeasureBlock(context, structType, &blockBytes, &openEnded, &tailStrideBytes) || + blockBytes % kWordBytes != 0 || (!openEnded && blockBytes == 0)) { MGLOG_D("[spirv] storage block %%%u holds a double but its byte layout cannot be " "described exactly; left to the fp64 demotion", structId); @@ -960,11 +1120,15 @@ namespace MobileGL { plan.structType = structType; plan.storageClass = storageClassByStruct[structId]; plan.wordCount = blockBytes / kWordBytes; + plan.openEnded = openEnded; + plan.tailStrideWords = tailStrideBytes / kWordBytes; + const uint32_t lastMember = structType->NumInOperands() - 1; bool expressible = true; for (Instruction* variable : variablesByStruct[structId]) { std::vector chains; std::unordered_set seenChains; + std::unordered_set seenLengths; defUseMgr->ForEachUser(variable, [&](Instruction* user) { if (!expressible) return; switch (user->opcode()) { @@ -982,6 +1146,27 @@ namespace MobileGL { } expressible = false; return; + case spv::Op::OpArrayLength: { + // Only an open-ended block has a length to ask for, and + // only of its last member; the result has to be the 32-bit + // uint the rewrite's arithmetic is typed in, which is the + // only result type the instruction allows anyway. + const Instruction* resultType = defUseMgr->GetDef(user->type_id()); + const bool isUint = resultType != nullptr && + resultType->opcode() == spv::Op::OpTypeInt && + resultType->GetSingleWordInOperand(0) == 32u && + resultType->GetSingleWordInOperand(1) == 0u; + if (openEnded && isUint && user->NumInOperands() >= 2 && + user->GetSingleWordInOperand(0) == variable->result_id() && + user->GetSingleWordInOperand(1) == lastMember) { + if (seenLengths.insert(user->result_id()).second) { + plan.arrayLengths.push_back(user); + } + return; + } + expressible = false; + return; + } default: expressible = false; return; @@ -1058,16 +1243,25 @@ namespace MobileGL { Emitter emitter(irContext, uintTypeId, boolTypeId, floatTypeId); bool modified = false; + // Every block declines before anything is written for it, so |touched| only ever + // parts company with |modified| on a shape that cannot happen without the module + // running out of ids - and even then the status must not claim the bytes are + // untouched, because the caller relies on that to skip invalidating its analyses. + bool touched = false; for (BlockPlan& plan : plans) { const uint32_t structId = plan.structType->result_id(); - const uint32_t arrayTypeId = - CreateWordArrayTypeBefore(irContext, plan.structType, uintTypeId, plan.wordCount); + const uint32_t arrayTypeId = CreateWordArrayTypeBefore( + irContext, plan.structType, uintTypeId, plan.wordCount, plan.openEnded); if (arrayTypeId == 0) { + // Nothing was written for it, so the module is still the one that came in. MGLOG_D("[spirv] storage block %%%u: no legal place for the flattened word array; " "left to the fp64 demotion", structId); continue; } + // Past this point the module HAS been written to, so an abandoned block would + // leave a dead type behind - the status has to say so even then. + touched = true; const uint32_t wordPointerTypeId = irContext->get_type_mgr()->FindPointerToType( uintTypeId, static_cast(plan.storageClass)); if (wordPointerTypeId == 0) continue; @@ -1095,6 +1289,9 @@ namespace MobileGL { } irContext->KillInst(chainPlan.chain); } + for (Instruction* arrayLength : plan.arrayLengths) { + emitter.RewriteArrayLength(arrayLength, plan.wordCount, plan.tailStrideWords); + } const std::vector surviving = SurvivingAccessQualifiers( irContext, structId, plan.structType->NumInOperands()); @@ -1113,12 +1310,19 @@ namespace MobileGL { {SPV_OPERAND_TYPE_DECORATION, {static_cast(kind)}}}); } modified = true; - MGLOG_D("[spirv] storage block %%%u: flattened into %u words so its 64-bit members keep " - "the byte layout the application bound", - structId, plan.wordCount); + if (plan.openEnded) { + MGLOG_D("[spirv] storage block %%%u: flattened into an open-ended word array (%u-word " + "prefix, %u-word elements) so its 64-bit members keep the byte layout the " + "application bound", + structId, plan.wordCount, plan.tailStrideWords); + } else { + MGLOG_D("[spirv] storage block %%%u: flattened into %u words so its 64-bit members " + "keep the byte layout the application bound", + structId, plan.wordCount); + } } - if (!modified) { + if (!modified && !touched) { return Status::SuccessWithoutChange; } irContext->InvalidateAnalysesExceptFor(IRContext::kAnalysisNone); diff --git a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h index c21dfd5f..f6c8c1fc 100644 --- a/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h +++ b/MobileGL/MG_Util/ShaderTranspiler/SpirvPasses/FlattenFloat64StorageBlockPass.h @@ -66,17 +66,32 @@ namespace MobileGL { // fp32 promise DemoteFloat64Pass already makes - what changes is only that the // BYTES around the value stay where the application put them. // + // AN OPEN-ENDED BLOCK - one whose last member is a runtime array, the + // `buffer B { double data[]; }` every unsized storage buffer is spelled as - is + // flattened the same way: the members before the array are the fixed prefix, and + // the flattened member is itself a `uint[]` runtime array, ArrayStride 4, with no + // length for the driver to re-derive. Element i of the original array lives at + // word `prefix + i * stride` of it, which is where the application put it. The + // block's `.length()` is rewritten too, because OpArrayLength on the flattened + // member counts WORDS: it becomes `(words - prefix) / stride` in unsigned + // arithmetic, clamped at zero when the bound range is shorter than the prefix, + // which is the floor GL defines `.length()` as. + // // DECLINES, leaving the block exactly as it was for DemoteFloat64Pass to handle the // old way, whenever it meets something it cannot rewrite exactly: // - a block whose variable is used as anything but an access-chain base (loaded - // whole, handed to a function, asked its OpArrayLength); + // whole, handed to a function), or asked an OpArrayLength it is not open-ended + // for; // - an access chain that is not rooted at the variable, or whose result feeds // anything but a plain OpLoad / OpStore (an atomic, OpCopyMemory, a further - // chain); - // - a non-constant index into a struct, a runtime array anywhere in the block, a - // RowMajor matrix (its columns are not contiguous, so a whole-column access is - // not one range), a member width other than 32 or 64 bits, or an offset or - // stride that is not a multiple of 4; + // chain), or one that names a whole runtime array rather than an element of it; + // - a non-constant index into a struct, a runtime array that is not the last + // member of the block itself (nested in a member, or followed by another - + // shapes GLSL cannot spell but SPIR-V can), a runtime array without an + // ArrayStride or whose element the pass cannot decompose, a RowMajor matrix (its + // columns are not contiguous, so a whole-column access is not one range), a + // member width other than 32 or 64 bits, or an offset or stride that is not a + // multiple of 4; // - a load or store whose type decomposes into more scalars than the cap below, // so legalizing a block can never explode the module. // diff --git a/tools/cts/scripts/run_cts.py b/tools/cts/scripts/run_cts.py index e322eb55..b2b7862b 100644 --- a/tools/cts/scripts/run_cts.py +++ b/tools/cts/scripts/run_cts.py @@ -51,6 +51,48 @@ def wait_for_device(serial, attempts=20, delay=15): return False +def device_file_size(serial, path): + r = adb(serial, "shell", f"stat -c %s {path} 2>/dev/null || echo 0", timeout=30) + m = re.search(r"(\d+)", r.stdout or "") + return int(m.group(1)) if m else 0 + + +def run_chunk(serial, cmd, dev_qpa, dev_list, idle_timeout, poll_interval=15): + """Run one glcts invocation; give up only when the log stops growing. + + A chunk is thousands of cases and legitimately runs for an hour, so a fixed + wall-clock cap would kill healthy invocations and record whichever case was + in flight as a crash. A GPU hang, by contrast, stops the .qpa from growing. + The timeout is therefore measured from the last observed growth of the + device-side log. On expiry the device-side glcts is killed (matched by the + caselist path this runner alone uses, so other processes are left alone) and + returncode 124 is reported, the same signal a hard timeout used to give. + """ + proc = subprocess.Popen(["adb", "-s", serial, "shell", cmd], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + last_size = -1 + last_growth = time.time() + while True: + try: + out, err = proc.communicate(timeout=poll_interval) + return subprocess.CompletedProcess(proc.args, proc.returncode, out, err) + except subprocess.TimeoutExpired: + pass + size = device_file_size(serial, dev_qpa) + now = time.time() + if size != last_size: + last_size = size + last_growth = now + elif now - last_growth > idle_timeout: + adb(serial, "shell", f"pkill -f {dev_list}", timeout=30) + proc.kill() + try: + proc.communicate(timeout=30) + except subprocess.TimeoutExpired: + pass + return subprocess.CompletedProcess(proc.args, 124, "", "idle timeout") + + def mem_available_kb(serial): r = adb(serial, "shell", "grep MemAvailable /proc/meminfo", timeout=30) m = re.search(r"(\d+)", r.stdout or "") @@ -152,7 +194,8 @@ def main(): ap.add_argument("--min-mem-kb", type=int, default=400000, help="pause when the device drops below this much available memory") ap.add_argument("--chunk-timeout", type=int, default=900, - help="seconds before giving up on one glcts invocation (a GPU hang never returns)") + help="seconds without any growth of the device-side .qpa before the glcts " + "invocation is declared hung and killed (a GPU hang never returns)") ap.add_argument("--skip-file", default=None, help="file of case names to exclude, e.g. cases known to hang the device") ap.add_argument("--env", action="append", default=[], metavar="K=V", @@ -239,10 +282,10 @@ def main(): f"--deqp-log-images=disable --deqp-log-shader-sources=disable " f"--deqp-log-filename={dev_qpa} > /dev/null 2>&1; rc=$?; sync; echo RC=$rc" ) - run = adb(args.serial, "shell", cmd, timeout=args.chunk_timeout) + run = run_chunk(args.serial, cmd, dev_qpa, dev_list, args.chunk_timeout) if run.returncode == 124: - print(f"[run_cts] chunk {chunk:04d} timed out after {args.chunk_timeout}s " - f"(likely a GPU hang)", file=sys.stderr) + print(f"[run_cts] chunk {chunk:04d}: no log growth for {args.chunk_timeout}s " + f"(likely a GPU hang); killed glcts", file=sys.stderr) # Some cases hang the GPU hard enough to reboot the device. The log on # /data/local/tmp survives that, so wait for the device to come back and