Add sharpening option to TAA

We use the RCAS algorithm from FSR1. This is useful for when TAA upscaling is enabled and similar to to what FSR2 is doing.
google · Jan 19, 2024 · ee6f3fb · ee6f3fb
1 parent b9a9586
commit ee6f3fb
Show file tree

Hide file tree

Showing 12 changed files with 103 additions and 54 deletions.
diff --git a/android/filament-android/src/main/java/com/google/android/filament/View.java b/android/filament-android/src/main/java/com/google/android/filament/View.java
@@ -1940,6 +1940,10 @@ public enum JitterPattern {
          * texturing lod bias (typically -1 or -2)
          */
         public float lodBias = -1.0f;
+        /**
+         * post-TAA sharpen, especially useful when upscaling is true.
+         */
+        public float sharpness = 0.0f;
         /**
          * enables or disables temporal anti-aliasing
          */

diff --git a/filament/include/filament/Options.h b/filament/include/filament/Options.h
@@ -438,6 +438,7 @@ struct TemporalAntiAliasingOptions {
     float filterWidth = 1.0f;   //!< reconstruction filter width typically between 0.2 (sharper, aliased) and 1.5 (smoother)
     float feedback = 0.12f;     //!< history feedback, between 0 (maximum temporal AA) and 1 (no temporal AA).
     float lodBias = -1.0f;      //!< texturing lod bias (typically -1 or -2)
+    float sharpness = 0.0f;     //!< post-TAA sharpen, especially useful when upscaling is true.
     bool enabled = false;       //!< enables or disables temporal anti-aliasing
     bool upscaling = false;     //!< 4x TAA upscaling. Disables Dynamic Resolution. [BETA]
 

diff --git a/filament/src/PostProcessManager.cpp b/filament/src/PostProcessManager.cpp
@@ -115,7 +115,7 @@ PostProcessManager::PostProcessMaterial::PostProcessMaterial() noexcept {
 }
 
 PostProcessManager::PostProcessMaterial::PostProcessMaterial(MaterialInfo const& info) noexcept
-    : PostProcessMaterial() {
+        : PostProcessMaterial() {
     mData = info.data; // aliased to mMaterial
     mSize = info.size;
     mConstants = info.constants;
@@ -167,7 +167,7 @@ void PostProcessManager::PostProcessMaterial::loadMaterial(FEngine& engine) cons
     mHasMaterial = true;
     auto builder = Material::Builder();
     builder.package(mData, mSize);
-    for (auto const& constant : mConstants) {
+    for (auto const& constant: mConstants) {
         std::visit([&](auto&& arg) {
             builder.constant(constant.name.data(), constant.name.size(), arg);
         }, constant.value);
@@ -189,14 +189,15 @@ PipelineState PostProcessManager::PostProcessMaterial::getPipelineState(
     FMaterial* const material = getMaterial(engine);
     material->prepareProgram(Variant{ variantKey });
     return {
-            .program = material->getProgram(Variant{variantKey}),
+            .program = material->getProgram(Variant{ variantKey }),
             .rasterState = material->getRasterState(),
             .scissor = material->getDefaultInstance()->getScissor()
     };
 }
 
 UTILS_NOINLINE
-FMaterialInstance* PostProcessManager::PostProcessMaterial::getMaterialInstance(FEngine& engine) const noexcept {
+FMaterialInstance* PostProcessManager::PostProcessMaterial::getMaterialInstance(
+        FEngine& engine) const noexcept {
     FMaterial* const material = getMaterial(engine);
     return material->getDefaultInstance();
 }
@@ -233,14 +234,15 @@ const PostProcessManager::JitterSequence<32>
 
 PostProcessManager::PostProcessManager(FEngine& engine) noexcept
         : mEngine(engine),
-         mWorkaroundSplitEasu(false),
-         mWorkaroundAllowReadOnlyAncillaryFeedbackLoop(false) {
+          mWorkaroundSplitEasu(false),
+          mWorkaroundAllowReadOnlyAncillaryFeedbackLoop(false) {
 }
 
 PostProcessManager::~PostProcessManager() noexcept = default;
 
 UTILS_NOINLINE
-void PostProcessManager::registerPostProcessMaterial(std::string_view name, MaterialInfo const& info) {
+void PostProcessManager::registerPostProcessMaterial(std::string_view name,
+        MaterialInfo const& info) {
     mMaterialRegistry.try_emplace(name, info);
 }
 
@@ -323,13 +325,13 @@ void PostProcessManager::init() noexcept {
             driver.isWorkaroundNeeded(Workaround::ALLOW_READ_ONLY_ANCILLARY_FEEDBACK_LOOP);
 
     #pragma nounroll
-    for (auto const& info : sMaterialListFeatureLevel0) {
+    for (auto const& info: sMaterialListFeatureLevel0) {
         registerPostProcessMaterial(info.name, info);
     }
 
     if (mEngine.getActiveFeatureLevel() >= FeatureLevel::FEATURE_LEVEL_1) {
         #pragma nounroll
-        for (auto const& info : sMaterialList) {
+        for (auto const& info: sMaterialList) {
             registerPostProcessMaterial(info.name, info);
         }
     }
@@ -2767,24 +2769,77 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::taa(FrameGraph& fg,
             });
 
     input = colorGradingConfig.asSubpass ? taaPass->tonemappedOutput : taaPass->output;
+    auto history = input;
+
+    // optional sharpen pass from FSR1
+    if (taaOptions.sharpness > 0.0f) {
+        input = rcas(fg, taaOptions.sharpness,
+                input, fg.getDescriptor(input), colorGradingConfig.translucent);
+    }
 
     struct ExportColorHistoryData {
         FrameGraphId<FrameGraphTexture> color;
     };
-    auto& exportHistoryPass = fg.addPass<ExportColorHistoryData>("Export TAA history",
+    fg.addPass<ExportColorHistoryData>("Export TAA history",
             [&](FrameGraph::Builder& builder, auto& data) {
                 // We need to use sideEffect here to ensure this pass won't be culled.
                 // The "output" of this pass is going to be used during the next frame as
                 // an "import".
                 builder.sideEffect();
-                data.color = builder.sample(input); // FIXME: an access must be declared for detach(), why?
-            }, [&current](FrameGraphResources const& resources, auto const& data,
-                    backend::DriverApi&) {
-                resources.detach(data.color,
-                        &current.color, &current.desc);
+                data.color = builder.sample(history); // FIXME: an access must be declared for detach(), why?
+            }, [&current](FrameGraphResources const& resources, auto const& data, auto&) {
+                resources.detach(data.color, &current.color, &current.desc);
+            });
+
+    return input;
+}
+
+FrameGraphId<FrameGraphTexture> PostProcessManager::rcas(
+        FrameGraph& fg,
+        float sharpness,
+        FrameGraphId<FrameGraphTexture> input,
+        FrameGraphTexture::Descriptor const& outDesc,
+        bool translucent) {
+
+    struct QuadBlitData {
+        FrameGraphId<FrameGraphTexture> input;
+        FrameGraphId<FrameGraphTexture> output;
+    };
+
+    auto& ppFsrRcas = fg.addPass<QuadBlitData>("FidelityFX FSR1 Rcas",
+            [&](FrameGraph::Builder& builder, auto& data) {
+                data.input = builder.sample(input);
+                data.input = builder.createTexture("FFX FSR1 Rcas output", outDesc);
+                data.input = builder.declareRenderPass(data.input);
+            },
+            [=](FrameGraphResources const& resources,
+                    auto const& data, DriverApi& driver) {
+
+                auto input = resources.getTexture(data.input);
+                auto out = resources.getRenderPassInfo();
+                auto const& outputDesc = resources.getDescriptor(data.input);
+
+                auto& material = getPostProcessMaterial("fsr_rcas");
+                FMaterialInstance* const mi = material.getMaterialInstance(mEngine);
+
+                FSRUniforms uniforms;
+                FSR_SharpeningSetup(&uniforms, { .sharpness = 2.0f - 2.0f * sharpness });
+                mi->setParameter("RcasCon", uniforms.RcasCon);
+                mi->setParameter("color", input, {}); // uses texelFetch
+                mi->setParameter("resolution", float4{
+                        outputDesc.width, outputDesc.height,
+                        1.0f / outputDesc.width, 1.0f / outputDesc.height });
+                mi->commit(driver);
+                mi->use(driver);
+
+                const uint8_t variant = uint8_t(
+                        translucent ? PostProcessVariant::TRANSLUCENT : PostProcessVariant::OPAQUE);
+
+                PipelineState const pipeline(material.getPipelineState(mEngine, variant));
+                render(out, pipeline, driver);
             });
 
-    return exportHistoryPass->color;
+    return ppFsrRcas->output;
 }
 
 FrameGraphId<FrameGraphTexture> PostProcessManager::upscale(FrameGraph& fg, bool translucent,
@@ -2947,41 +3002,9 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::upscale(FrameGraph& fg, bool
     auto output = ppQuadBlit->output;
 
     // if we had to take the low quality fallback, we still do the "sharpen pass"
-    if (dsrOptions.sharpness > 0.0f && (dsrOptions.quality != QualityLevel::LOW || lowQualityFallback)) {
-        auto& ppFsrRcas = fg.addPass<QuadBlitData>("FidelityFX FSR1 Rcas",
-                [&](FrameGraph::Builder& builder, auto& data) {
-                    data.input = builder.sample(output);
-                    data.output = builder.createTexture("FFX FSR1 Rcas output", outDesc);
-                    data.output = builder.declareRenderPass(data.output);
-                },
-                [=](FrameGraphResources const& resources,
-                        auto const& data, DriverApi& driver) {
-
-                    auto color = resources.getTexture(data.input);
-                    auto out = resources.getRenderPassInfo();
-                    auto const& outputDesc = resources.getDescriptor(data.output);
-
-                    auto& material = getPostProcessMaterial("fsr_rcas");
-                    FMaterialInstance* const mi = material.getMaterialInstance(mEngine);
-
-                    FSRUniforms uniforms;
-                    FSR_SharpeningSetup(&uniforms, { .sharpness = 2.0f - 2.0f * dsrOptions.sharpness });
-                    mi->setParameter("RcasCon", uniforms.RcasCon);
-                    mi->setParameter("color", color, { }); // uses texelFetch
-                    mi->setParameter("resolution", float4{
-                            outputDesc.width, outputDesc.height,
-                            1.0f / outputDesc.width, 1.0f / outputDesc.height });
-                    mi->commit(driver);
-                    mi->use(driver);
-
-                    const uint8_t variant = uint8_t(translucent ?
-                            PostProcessVariant::TRANSLUCENT : PostProcessVariant::OPAQUE);
-
-                    PipelineState const pipeline(material.getPipelineState(mEngine, variant));
-                    render(out, pipeline, driver);
-                });
-
-        output = ppFsrRcas->output;
+    if (dsrOptions.sharpness > 0.0f &&
+            (dsrOptions.quality != QualityLevel::LOW || lowQualityFallback)) {
+        output = rcas(fg, dsrOptions.sharpness, output, outDesc, translucent);
     }
 
     // we rely on automatic culling of unused render passes

diff --git a/filament/src/PostProcessManager.h b/filament/src/PostProcessManager.h
@@ -242,6 +242,13 @@ class PostProcessManager {
             filament::Viewport const& vp, FrameGraphTexture::Descriptor const& outDesc,
             backend::SamplerMagFilter filter) noexcept;
 
+    FrameGraphId<FrameGraphTexture> rcas(
+            FrameGraph& fg,
+            float sharpness,
+            FrameGraphId<FrameGraphTexture> input,
+            FrameGraphTexture::Descriptor const& outDesc,
+            bool translucent);
+
     // upscale/downscale blitter using shaders
     FrameGraphId<FrameGraphTexture> blit(FrameGraph& fg, bool translucent,
             FrameGraphId<FrameGraphTexture> input,

diff --git a/filament/src/details/Renderer.cpp b/filament/src/details/Renderer.cpp
@@ -985,8 +985,7 @@ void FRenderer::renderJob(ArenaScope& arena, FView& view) {
                     auto& history = view.getFrameHistory();
                     auto& current = history.getCurrent();
                     current.ssr.projection = projection;
-                    resources.detach(data.history,
-                            &current.ssr.color, &current.ssr.desc);
+                    resources.detach(data.history, &current.ssr.color, &current.ssr.desc);
                 });
     }
 

diff --git a/filament/src/fg/details/Resource.h b/filament/src/fg/details/Resource.h
@@ -132,7 +132,7 @@ class Resource : public VirtualResource {
     Descriptor descriptor;
     SubResourceDescriptor subResourceDescriptor;
 
-    // weather the resource was detached
+    // whether the resource was detached
     bool detached = false;
 
     // An Edge with added data from this resource

diff --git a/filament/src/materials/fsr/fsr_rcas.mat b/filament/src/materials/fsr/fsr_rcas.mat
@@ -43,6 +43,8 @@ fragment {
     #define FSR_RCAS_F 1
     #if !POST_PROCESS_OPAQUE
     #   define FSR_RCAS_PASSTHROUGH_ALPHA
+    // todo: make this a spec constant
+    #   define FSR_RCAS_DENOISE
     #endif
     #include "ffx_fsr1.h"
 
@@ -61,6 +63,9 @@ fragment {
 #endif
             p, materialParams.RcasCon);
 
+        // todo: make this an option
+        postProcess.color.rgb = max(vec3(0), postProcess.color.rgb);
+
 #if POST_PROCESS_OPAQUE
         postProcess.color.a = 1.0;
 #endif

diff --git a/libs/viewer/src/Settings_generated.cpp b/libs/viewer/src/Settings_generated.cpp
@@ -723,6 +723,8 @@ int parse(jsmntok_t const* tokens, int i, const char* jsonChunk, TemporalAntiAli
             i = parse(tokens, i + 1, jsonChunk, &out->feedback);
         } else if (compare(tok, jsonChunk, "lodBias") == 0) {
             i = parse(tokens, i + 1, jsonChunk, &out->lodBias);
+        } else if (compare(tok, jsonChunk, "sharpness") == 0) {
+            i = parse(tokens, i + 1, jsonChunk, &out->sharpness);
         } else if (compare(tok, jsonChunk, "enabled") == 0) {
             i = parse(tokens, i + 1, jsonChunk, &out->enabled);
         } else if (compare(tok, jsonChunk, "upscaling") == 0) {
@@ -762,6 +764,7 @@ std::ostream& operator<<(std::ostream& out, const TemporalAntiAliasingOptions& i
         << "\"filterWidth\": " << (in.filterWidth) << ",\n"
         << "\"feedback\": " << (in.feedback) << ",\n"
         << "\"lodBias\": " << (in.lodBias) << ",\n"
+        << "\"sharpness\": " << (in.sharpness) << ",\n"
         << "\"enabled\": " << to_string(in.enabled) << ",\n"
         << "\"upscaling\": " << to_string(in.upscaling) << ",\n"
         << "\"filterHistory\": " << to_string(in.filterHistory) << ",\n"

diff --git a/libs/viewer/src/ViewerGui.cpp b/libs/viewer/src/ViewerGui.cpp
@@ -809,6 +809,7 @@ void ViewerGui::updateUserInterface() {
         ImGui::Combo("Box Clipping", &boxClipping, "Accurate\0Clamp\0None\0\0");
         ImGui::Combo("Box Type", &boxType, "AABB\0Variance\0Both\0\0");
         ImGui::SliderFloat("Variance Gamma", &mSettings.view.taa.varianceGamma, 0.75f, 1.25f);
+        ImGui::SliderFloat("RCAS", &mSettings.view.taa.sharpness, 0.0f, 1.0f);
         mSettings.view.taa.boxClipping = (TemporalAntiAliasingOptions::BoxClipping)boxClipping;
         mSettings.view.taa.boxType = (TemporalAntiAliasingOptions::BoxType)boxType;
         mSettings.view.taa.jitterPattern = (TemporalAntiAliasingOptions::JitterPattern)jitterSequence;

diff --git a/web/filament-js/extensions_generated.js b/web/filament-js/extensions_generated.js
@@ -141,6 +141,7 @@ Filament.loadGeneratedExtensions = function() {
             filterWidth: 1.0,
             feedback: 0.12,
             lodBias: -1.0,
+            sharpness: 0.0,
             enabled: false,
             upscaling: false,
             filterHistory: true,

diff --git a/web/filament-js/filament.d.ts b/web/filament-js/filament.d.ts
@@ -1684,6 +1684,10 @@ export interface View$TemporalAntiAliasingOptions {
      * texturing lod bias (typically -1 or -2)
      */
     lodBias?: number;
+    /**
+     * post-TAA sharpen, especially useful when upscaling is true.
+     */
+    sharpness?: number;
     /**
      * enables or disables temporal anti-aliasing
      */

diff --git a/web/filament-js/jsbindings_generated.cpp b/web/filament-js/jsbindings_generated.cpp
@@ -121,6 +121,7 @@ value_object<View::TemporalAntiAliasingOptions>("View$TemporalAntiAliasingOption
     .field("filterWidth", &View::TemporalAntiAliasingOptions::filterWidth)
     .field("feedback", &View::TemporalAntiAliasingOptions::feedback)
     .field("lodBias", &View::TemporalAntiAliasingOptions::lodBias)
+    .field("sharpness", &View::TemporalAntiAliasingOptions::sharpness)
     .field("enabled", &View::TemporalAntiAliasingOptions::enabled)
     .field("upscaling", &View::TemporalAntiAliasingOptions::upscaling)
     .field("filterHistory", &View::TemporalAntiAliasingOptions::filterHistory)