/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.axis2.json.streaming;

import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import org.apache.axiom.om.OMAbstractFactory;
import org.apache.axiom.om.OMOutputFormat;
import org.apache.axiom.soap.SOAPFactory;
import org.apache.axis2.Constants;
import org.apache.axis2.context.MessageContext;
import org.apache.axis2.json.factory.JsonConstant;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;

/**
 * Unit tests for {@link FieldFilteringMessageFormatter} with
 * {@link MoshiStreamingMessageFormatter} as the delegate.
 *
 * <p>Option C architecture: field filtering happens inside the Moshi
 * serialization layer via reflection-based selective field writing.
 * The streaming pipeline (Moshi → Okio → FlushingOutputStream) is
 * never broken — no capture buffer is used.</p>
 */
public class FieldFilteringMessageFormatterTest {

    protected MessageContext outMsgContext;
    protected OMOutputFormat outputFormat;
    protected ByteArrayOutputStream outputStream;
    protected FieldFilteringMessageFormatter formatter;

    @Before
    public void setUp() {
        SOAPFactory soapFactory = OMAbstractFactory.getSOAP11Factory();
        outputFormat = new OMOutputFormat();
        outputStream = new ByteArrayOutputStream();

        outMsgContext = new MessageContext();
        outMsgContext.setProperty(Constants.Configuration.CHARACTER_SET_ENCODING, "UTF-8");

        formatter = new FieldFilteringMessageFormatter(
            new MoshiStreamingMessageFormatter());
    }

    @After
    public void tearDown() throws Exception {
        outputStream.close();
    }

    // ── Core filtering (through real streaming pipeline) ──────────────────

    @Test
    public void testFilterKeepsSelectedFields() throws Exception {
        setReturnObject(new SampleData("SUCCESS", 0.025, 0.157, 1));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "variance"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        Assert.assertTrue(response.getAsJsonObject().has("status"));
        Assert.assertTrue(response.getAsJsonObject().has("variance"));
        Assert.assertFalse("volatility should be filtered",
            response.getAsJsonObject().has("volatility"));
        Assert.assertFalse("calcTimeUs should be filtered",
            response.getAsJsonObject().has("calcTimeUs"));
    }

    @Test
    public void testFilterWithAllFieldsMatchesUnfiltered() throws Exception {
        SampleData data = new SampleData("SUCCESS", 0.025, 0.157, 1);
        setReturnObject(data);

        // Unfiltered baseline
        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        String unfiltered = outputStream.toString("UTF-8");

        // Filtered with ALL fields
        outputStream.reset();
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "variance", "volatility", "calcTimeUs"));
        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        String filtered = outputStream.toString("UTF-8");

        JsonElement expected = JsonParser.parseString(unfiltered);
        JsonElement actual = JsonParser.parseString(filtered);
        Assert.assertEquals(expected, actual);
    }

    @Test
    public void testFilterWithNoMatchingFieldsProducesEmptyResponse() throws Exception {
        setReturnObject(new SampleData("SUCCESS", 0.025, 0.157, 1));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("nonexistent"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        Assert.assertEquals("Empty response object", 0,
            response.getAsJsonObject().size());
    }

    @Test
    public void testFilterPreservesNestedObjects() throws Exception {
        setReturnObject(new NestedData("SUCCESS", new InnerData(42, "hello"), 99));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("nested"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        Assert.assertTrue("nested should be kept",
            response.getAsJsonObject().has("nested"));
        Assert.assertFalse("status should be filtered",
            response.getAsJsonObject().has("status"));
        Assert.assertFalse("flat should be filtered",
            response.getAsJsonObject().has("flat"));
        // Verify nested structure intact
        Assert.assertEquals(42,
            response.getAsJsonObject().getAsJsonObject("nested")
                .get("value").getAsInt());
    }

    @Test
    public void testFilterPreservesArrayFields() throws Exception {
        setReturnObject(new ArrayData("SUCCESS", Arrays.asList(1, 2, 3), 3));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("items"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        Assert.assertTrue("items should be kept",
            response.getAsJsonObject().has("items"));
        Assert.assertEquals(3,
            response.getAsJsonObject().getAsJsonArray("items").size());
        Assert.assertFalse("count should be filtered",
            response.getAsJsonObject().has("count"));
    }

    @Test
    public void testFilterPreservesNullValues() throws Exception {
        setReturnObject(new NullableData("SUCCESS", null, 42));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "error"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        Assert.assertTrue(response.getAsJsonObject().has("status"));
        Assert.assertTrue("null field should be kept",
            response.getAsJsonObject().has("error"));
        Assert.assertTrue(response.getAsJsonObject().get("error").isJsonNull());
        Assert.assertFalse("val should be filtered",
            response.getAsJsonObject().has("val"));
    }

    // ── Delegate passthrough tests ────────────────────────────────────────

    @Test
    public void testNoFilterDelegatesDirectly() throws Exception {
        setReturnObject(new SampleData("SUCCESS", 0.025, 0.157, 1));
        // No FIELD_FILTER set

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        String result = outputStream.toString("UTF-8");

        Assert.assertTrue(result.contains("\"status\":\"SUCCESS\""));
        Assert.assertTrue(result.contains("\"variance\""));
        Assert.assertTrue(result.contains("\"volatility\""));
        Assert.assertTrue(result.contains("\"calcTimeUs\""));
    }

    @Test
    public void testFaultResponseNeverFiltered() throws Exception {
        outMsgContext.setProcessingFault(true);
        SOAPFactory soapFactory = OMAbstractFactory.getSOAP11Factory();
        var envelope = soapFactory.getDefaultEnvelope();
        envelope.getBody().addChild(TestHelper.createFaultElement());
        outMsgContext.setEnvelope(envelope);
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("nonexistent"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        String result = outputStream.toString("UTF-8");

        Assert.assertTrue("Fault should pass through",
            result.contains("Fault"));
        Assert.assertTrue(result.contains("faultcode"));
        Assert.assertTrue(result.contains("faultstring"));
    }

    // ── Inheritance and modifier edge cases ──────────────────────────────

    @Test
    public void testFilterIncludesInheritedFields() throws Exception {
        setReturnObject(new ChildData("SUCCESS", 0.025, "extra-info"));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "extra"));  // status is inherited, extra is declared

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        Assert.assertTrue("inherited 'status' should be included",
            response.getAsJsonObject().has("status"));
        Assert.assertTrue("declared 'extra' should be included",
            response.getAsJsonObject().has("extra"));
        Assert.assertFalse("inherited 'variance' should be filtered",
            response.getAsJsonObject().has("variance"));
    }

    @Test
    public void testFilterIncludesPrivateFields() throws Exception {
        setReturnObject(new PrivateFieldData("secret-val", 99));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("secret"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        Assert.assertTrue("private 'secret' should be included",
            response.getAsJsonObject().has("secret"));
        Assert.assertEquals("secret-val",
            response.getAsJsonObject().get("secret").getAsString());
        Assert.assertFalse("private 'value' should be filtered",
            response.getAsJsonObject().has("value"));
    }

    @Test
    public void testStaticAndTransientFieldsNeverIncluded() throws Exception {
        setReturnObject(new StaticTransientData("SUCCESS", 42));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "value", "CONSTANT", "tempCache"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        Assert.assertTrue("instance 'status' should be included",
            response.getAsJsonObject().has("status"));
        Assert.assertTrue("instance 'value' should be included",
            response.getAsJsonObject().has("value"));
        Assert.assertFalse("static 'CONSTANT' must never appear",
            response.getAsJsonObject().has("CONSTANT"));
        Assert.assertFalse("transient 'tempCache' must never appear",
            response.getAsJsonObject().has("tempCache"));
    }

    // ── parseFieldsCsv / parseFieldsFromUrl tests ─────────────────────────

    @Test
    public void testParseFieldsCsv() {
        Set<String> fields = FieldFilteringMessageFormatter.parseFieldsCsv(
            "status, variance , calcTimeUs");
        Assert.assertEquals(3, fields.size());
        Assert.assertTrue(fields.contains("status"));
        Assert.assertTrue(fields.contains("variance"));
        Assert.assertTrue(fields.contains("calcTimeUs"));
    }

    @Test
    public void testParseFieldsCsvEmpty() {
        Assert.assertTrue(
            FieldFilteringMessageFormatter.parseFieldsCsv("").isEmpty());
        Assert.assertTrue(
            FieldFilteringMessageFormatter.parseFieldsCsv(null).isEmpty());
    }

    @Test
    public void testParseFieldsFromUrl() {
        Set<String> fields = FieldFilteringMessageFormatter.parseFieldsFromUrl(
            "https://host/services/Svc?fields=status,variance&other=1");
        Assert.assertNotNull(fields);
        Assert.assertEquals(2, fields.size());
        Assert.assertTrue(fields.contains("status"));
        Assert.assertTrue(fields.contains("variance"));
    }

    @Test
    public void testParseFieldsFromUrlEncoded() {
        Set<String> fields = FieldFilteringMessageFormatter.parseFieldsFromUrl(
            "https://host/services/Svc?fields=first%20name,last%20name");
        Assert.assertNotNull(fields);
        Assert.assertEquals(2, fields.size());
        Assert.assertTrue(fields.contains("first name"));
        Assert.assertTrue(fields.contains("last name"));
    }

    @Test
    public void testParseFieldsCsvDuplicates() {
        Set<String> fields = FieldFilteringMessageFormatter.parseFieldsCsv(
            "status,variance,status");
        Assert.assertEquals("Duplicates should be deduplicated", 2, fields.size());
    }

    @Test
    public void testParseFieldsCsvEmptyParam() {
        // ?fields= with empty value
        Set<String> fields = FieldFilteringMessageFormatter.parseFieldsFromUrl(
            "https://host/services/Svc?fields=");
        Assert.assertNotNull(fields);
        Assert.assertTrue("Empty fields= should produce empty set", fields.isEmpty());
    }

    @Test
    public void testParseFieldsFromUrlNoParam() {
        Assert.assertNull(
            FieldFilteringMessageFormatter.parseFieldsFromUrl(
                "https://host/services/Svc?other=1"));
        Assert.assertNull(
            FieldFilteringMessageFormatter.parseFieldsFromUrl(
                "https://host/services/Svc"));
        Assert.assertNull(
            FieldFilteringMessageFormatter.parseFieldsFromUrl(null));
    }

    // ── Helpers ───────────────────────────────────────────────────────────

    private void setReturnObject(Object obj) {
        outMsgContext.setProperty(JsonConstant.RETURN_OBJECT, obj);
        outMsgContext.setProperty(JsonConstant.RETURN_TYPE, obj.getClass());
    }

    private JsonElement parseResponse() {
        String json;
        try {
            json = outputStream.toString("UTF-8");
        } catch (java.io.UnsupportedEncodingException e) {
            throw new AssertionError("UTF-8 encoding not supported", e);
        }
        JsonElement root = JsonParser.parseString(json);
        return root.getAsJsonObject().get("response");
    }

    private static Set<String> setOf(String... values) {
        return new LinkedHashSet<>(Arrays.asList(values));
    }

    // ── Test POJOs ────────────────────────────────────────────────────────

    public static class SampleData {
        public String status;
        public double variance;
        public double volatility;
        public long calcTimeUs;
        public SampleData() {}
        public SampleData(String s, double v, double vol, long t) {
            status = s; variance = v; volatility = vol; calcTimeUs = t;
        }
    }

    public static class InnerData {
        public int value;
        public String label;
        public InnerData() {}
        public InnerData(int v, String l) { value = v; label = l; }
    }

    public static class NestedData {
        public String status;
        public InnerData nested;
        public int flat;
        public NestedData() {}
        public NestedData(String s, InnerData n, int f) {
            status = s; nested = n; flat = f;
        }
    }

    public static class ArrayData {
        public String status;
        public List<Integer> items;
        public int count;
        public ArrayData() {}
        public ArrayData(String s, List<Integer> i, int c) {
            status = s; items = i; count = c;
        }
    }

    public static class NullableData {
        public String status;
        public String error;
        public int val;
        public NullableData() {}
        public NullableData(String s, String e, int v) {
            status = s; error = e; val = v;
        }
    }

    /** POJO with private fields for setAccessible test. */
    public static class PrivateFieldData {
        private String secret;
        private int value;
        public PrivateFieldData() {}
        public PrivateFieldData(String s, int v) { secret = s; value = v; }
    }

    /** Base class for inheritance test. */
    public static class BaseData {
        public String status;
        public double variance;
        public BaseData() {}
        public BaseData(String s, double v) { status = s; variance = v; }
    }

    /** Child that adds a field; inherits status + variance from BaseData. */
    public static class ChildData extends BaseData {
        public String extra;
        public ChildData() {}
        public ChildData(String s, double v, String e) {
            super(s, v); extra = e;
        }
    }

    /** POJO with static and transient fields for modifier filtering test. */
    public static class StaticTransientData {
        public static final String CONSTANT = "should-never-serialize";
        public transient String tempCache = "should-never-serialize";
        public String status;
        public int value;
        public StaticTransientData() {}
        public StaticTransientData(String s, int v) { status = s; value = v; }
    }

    // ── Large-payload field filtering (127-field POJO) ─────────────────
    //
    // Tests prove that ?fields= can select a small subset from a wide
    // response object (127 fields) without breaking serialization or
    // losing data. Uses programmatically generated generic field names.

    /**
     * POJO with 127 fields to exercise field filtering at scale.
     * Uses mixed types (String, double, int, long, boolean) across
     * all 127 fields to verify that filtering handles every
     * primitive and object type correctly.
     */
    public static class WideRecord {
        // 30 String fields
        public String s0, s1, s2, s3, s4, s5, s6, s7, s8, s9;
        public String s10, s11, s12, s13, s14, s15, s16, s17, s18, s19;
        public String s20, s21, s22, s23, s24, s25, s26, s27, s28, s29;
        // 40 double fields
        public double d0, d1, d2, d3, d4, d5, d6, d7, d8, d9;
        public double d10, d11, d12, d13, d14, d15, d16, d17, d18, d19;
        public double d20, d21, d22, d23, d24, d25, d26, d27, d28, d29;
        public double d30, d31, d32, d33, d34, d35, d36, d37, d38, d39;
        // 25 int fields
        public int i0, i1, i2, i3, i4, i5, i6, i7, i8, i9;
        public int i10, i11, i12, i13, i14, i15, i16, i17, i18, i19;
        public int i20, i21, i22, i23, i24;
        // 20 long fields
        public long l0, l1, l2, l3, l4, l5, l6, l7, l8, l9;
        public long l10, l11, l12, l13, l14, l15, l16, l17, l18, l19;
        // 12 boolean fields — total: 30+40+25+20+12 = 127
        public boolean b0, b1, b2, b3, b4, b5, b6, b7, b8, b9;
        public boolean b10, b11;

        public WideRecord() {}

        /** Populate all 127 fields with non-default values. */
        public static WideRecord createTestRecord() {
            WideRecord r = new WideRecord();
            for (int n = 0; n < 30; n++) {
                try {
                    r.getClass().getField("s" + n).set(r, "val_" + n);
                } catch (ReflectiveOperationException e) { throw new AssertionError("Field setup failed", e); }
            }
            for (int n = 0; n < 40; n++) {
                try {
                    r.getClass().getField("d" + n).setDouble(r, n * 1.1);
                } catch (ReflectiveOperationException e) { throw new AssertionError("Field setup failed", e); }
            }
            for (int n = 0; n < 25; n++) {
                try {
                    r.getClass().getField("i" + n).setInt(r, n * 100);
                } catch (ReflectiveOperationException e) { throw new AssertionError("Field setup failed", e); }
            }
            for (int n = 0; n < 20; n++) {
                try {
                    r.getClass().getField("l" + n).setLong(r, n * 1000000L);
                } catch (ReflectiveOperationException e) { throw new AssertionError("Field setup failed", e); }
            }
            for (int n = 0; n < 12; n++) {
                try {
                    r.getClass().getField("b" + n).setBoolean(r, n % 2 == 0);
                } catch (ReflectiveOperationException e) { throw new AssertionError("Field setup failed", e); }
            }
            return r;
        }
    }

    @Test
    public void testFilter127FieldsKeepOne() throws Exception {
        setReturnObject(WideRecord.createTestRecord());
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("s0"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        Assert.assertEquals("Should have exactly 1 field", 1,
            response.getAsJsonObject().size());
        Assert.assertEquals("val_0",
            response.getAsJsonObject().get("s0").getAsString());
    }

    @Test
    public void testFilter127FieldsKeepFive() throws Exception {
        setReturnObject(WideRecord.createTestRecord());
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("s0", "d5", "i10", "l15", "b0"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        Assert.assertEquals("Should have exactly 5 fields", 5,
            response.getAsJsonObject().size());
        Assert.assertEquals("val_0",
            response.getAsJsonObject().get("s0").getAsString());
        Assert.assertEquals(5.5,
            response.getAsJsonObject().get("d5").getAsDouble(), 0.0001);
        Assert.assertEquals(1000,
            response.getAsJsonObject().get("i10").getAsInt());
        Assert.assertEquals(15000000L,
            response.getAsJsonObject().get("l15").getAsLong());
        Assert.assertTrue(
            response.getAsJsonObject().get("b0").getAsBoolean());
    }

    @Test
    public void testFilter127FieldsPayloadSizeReduction() throws Exception {
        WideRecord record = WideRecord.createTestRecord();

        // Full response (all 127 fields)
        setReturnObject(record);
        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        int fullSize = outputStream.size();

        // Filtered response (1 field out of 127)
        outputStream.reset();
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("s0"));
        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        int filteredSize = outputStream.size();

        // The filtered response should be dramatically smaller
        double reductionPct = (1.0 - (double) filteredSize / fullSize) * 100;

        Assert.assertTrue(
            "Full response (" + fullSize + " bytes) should be > 1KB",
            fullSize > 1000);
        Assert.assertTrue(
            "Filtered response (" + filteredSize + " bytes) should be < 200 bytes",
            filteredSize < 200);
        Assert.assertTrue(
            "Payload reduction (" + String.format("%.0f", reductionPct)
                + "%) should exceed 90%",
            reductionPct > 90.0);
    }

    // ── Nested dot-notation field filtering ─────────────────────────────
    //
    // Models a service returning a top-level response with a nested array
    // of wide objects (127 fields each). Proves that dot-notation like
    // ?fields=status,records.s0 can filter inside each array element,
    // removing 126 of 127 fields per element without breaking the
    // streaming pipeline.

    /** Response wrapper: status + a list of WideRecord elements. */
    public static class ServiceResponse {
        public String status;
        public long calcTimeUs;
        public List<WideRecord> records;

        public ServiceResponse() {}
        public ServiceResponse(String status, long calcTimeUs, List<WideRecord> records) {
            this.status = status;
            this.calcTimeUs = calcTimeUs;
            this.records = records;
        }
    }

    /** Build a ServiceResponse with N records, each having 127 fields. */
    private static ServiceResponse buildNestedResponse(int nRecords) {
        List<WideRecord> records = new ArrayList<>();
        for (int i = 0; i < nRecords; i++) {
            WideRecord r = WideRecord.createTestRecord();
            // Give each record a unique s0 so we can verify identity
            try {
                r.getClass().getField("s0").set(r, "record_" + i);
            } catch (ReflectiveOperationException e) {
                throw new AssertionError("Failed to set s0", e);
            }
            records.add(r);
        }
        return new ServiceResponse("SUCCESS", 42, records);
    }

    @Test
    public void testNestedDotNotationKeepsOneFieldPerElement() throws Exception {
        // 3 records x 127 fields each. Filter to status + records.s0 only.
        // Each array element should have exactly 1 field (s0).
        setReturnObject(buildNestedResponse(3));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "records.s0"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        // Top level: status + records (calcTimeUs filtered out)
        Assert.assertEquals("SUCCESS",
            response.getAsJsonObject().get("status").getAsString());
        Assert.assertFalse("calcTimeUs should be filtered",
            response.getAsJsonObject().has("calcTimeUs"));

        // Each array element: only s0 remains
        var records = response.getAsJsonObject().getAsJsonArray("records");
        Assert.assertEquals(3, records.size());
        for (int i = 0; i < 3; i++) {
            var record = records.get(i).getAsJsonObject();
            Assert.assertEquals("record_" + i,
                record.get("s0").getAsString());
            Assert.assertEquals(
                "Element " + i + " should have exactly 1 field", 1,
                record.size());
        }
    }

    @Test
    public void testNestedDotNotationKeepsFiveFieldsPerElement() throws Exception {
        // Filter to 5 fields per element across different types
        setReturnObject(buildNestedResponse(2));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "records.s0", "records.d5",
                  "records.i10", "records.l15", "records.b0"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        var records = response.getAsJsonObject().getAsJsonArray("records");
        Assert.assertEquals(2, records.size());

        var first = records.get(0).getAsJsonObject();
        Assert.assertEquals("Should have exactly 5 fields per element", 5,
            first.size());
        Assert.assertEquals("record_0", first.get("s0").getAsString());
        Assert.assertEquals(5.5, first.get("d5").getAsDouble(), 0.0001);
        Assert.assertEquals(1000, first.get("i10").getAsInt());
        Assert.assertEquals(15000000L, first.get("l15").getAsLong());
        Assert.assertTrue(first.get("b0").getAsBoolean());
    }

    @Test
    public void testNestedDotNotation126of127FieldsRemoved() throws Exception {
        // The headline test: 127 fields per element, keep 1, verify massive
        // payload reduction on wide nested data structures.
        ServiceResponse full = buildNestedResponse(10);

        // Full response (all fields)
        setReturnObject(full);
        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        int fullSize = outputStream.size();

        // Filtered response: keep only records.s0 (1 of 127 per element)
        outputStream.reset();
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "records.s0"));
        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        int filteredSize = outputStream.size();

        double reductionPct = (1.0 - (double) filteredSize / fullSize) * 100;

        Assert.assertTrue(
            "Full response (" + fullSize + " bytes) should be > 10KB for 10 records",
            fullSize > 10000);
        Assert.assertTrue(
            "Filtered response (" + filteredSize + " bytes) should be < 500 bytes",
            filteredSize < 500);
        Assert.assertTrue(
            "Payload reduction (" + String.format("%.0f", reductionPct)
                + "%) should exceed 95%",
            reductionPct > 95.0);
    }

    @Test
    public void testNestedDotNotationNonexistentSubField() throws Exception {
        // Sub-field doesn't exist — array elements should be empty objects
        setReturnObject(buildNestedResponse(2));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "records.nonexistent"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        var records = response.getAsJsonObject().getAsJsonArray("records");
        Assert.assertEquals(2, records.size());
        Assert.assertEquals("Empty object when sub-field doesn't exist", 0,
            records.get(0).getAsJsonObject().size());
    }

    @Test
    public void testNoDotsPassesThrough() throws Exception {
        // Without dot-notation, the existing flat filtering behavior is unchanged.
        // "records" without dots keeps the entire array unfiltered.
        setReturnObject(buildNestedResponse(1));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "records"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        Assert.assertTrue(response.getAsJsonObject().has("records"));
        var first = response.getAsJsonObject().getAsJsonArray("records")
            .get(0).getAsJsonObject();
        // All 127 fields should be present (no nested filtering)
        Assert.assertTrue("All fields should be present without dot-notation",
            first.size() > 100);
    }

    // ── Multi-level dot-notation (Map-of-List-of-Map pattern) ──────────
    //
    // Models the real-world pattern where a service returns a response POJO
    // with a Map<String, Object> field ("data") that contains a
    // List<Map<String, Object>> ("records") where each map has 127 keys.
    // Two-level dot-notation: ?fields=status,data.records.s0

    /** Response with a Map<String, Object> field — the Map-based pattern. */
    public static class MapResponse {
        public String status;
        public long calcTimeMs;
        public java.util.Map<String, Object> data;

        public MapResponse() {}
        public MapResponse(String status, long calcTimeMs,
                           java.util.Map<String, Object> data) {
            this.status = status;
            this.calcTimeMs = calcTimeMs;
            this.data = data;
        }
    }

    /** Build a MapResponse with N calculation records (127 keys each)
     *  inside data.records, plus metadata keys in data. */
    private static MapResponse buildMapResponse(int nRecords) {
        java.util.Map<String, Object> data = new java.util.LinkedHashMap<>();

        // Records: list of maps, each with 127 keys
        List<java.util.Map<String, Object>> records = new ArrayList<>();
        for (int i = 0; i < nRecords; i++) {
            java.util.Map<String, Object> record = new java.util.LinkedHashMap<>();
            // 30 string fields
            for (int n = 0; n < 30; n++) record.put("s" + n, "val_" + i + "_" + n);
            // 40 double fields
            for (int n = 0; n < 40; n++) record.put("d" + n, n * 1.1 + i);
            // 25 int fields
            for (int n = 0; n < 25; n++) record.put("i" + n, n * 100 + i);
            // 20 long fields
            for (int n = 0; n < 20; n++) record.put("l" + n, (long)(n * 1000000 + i));
            // 12 boolean fields  — total: 30+40+25+20+12 = 127
            for (int n = 0; n < 12; n++) record.put("b" + n, n % 2 == 0);
            records.add(record);
        }
        data.put("records", records);

        // Metadata keys (would be filtered out in a real query)
        data.put("notes", Arrays.asList("note1", "note2"));
        data.put("diagnostics", java.util.Map.of("cacheHit", true, "queryTimeMs", 42));
        data.put("viewState", "some-large-view-state-blob");

        return new MapResponse("SUCCESS", 42, data);
    }

    @Test
    public void testTwoLevelDotNotationFiltersMapsRecursively() throws Exception {
        // Two-level: data.records.s0 — keep only s0 in each calculation
        setReturnObject(buildMapResponse(3));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "data.records.s0"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        // Top level: status + data (calcTimeMs filtered)
        Assert.assertTrue(response.getAsJsonObject().has("status"));
        Assert.assertTrue(response.getAsJsonObject().has("data"));
        Assert.assertFalse("calcTimeMs should be filtered",
            response.getAsJsonObject().has("calcTimeMs"));

        // data: only "records" key (notes, diagnostics, viewState filtered)
        var data = response.getAsJsonObject().getAsJsonObject("data");
        Assert.assertEquals("data should have exactly 1 key", 1, data.size());
        Assert.assertTrue(data.has("records"));

        // Each calculation element: only s0 (126 of 127 keys filtered)
        var calcs = data.getAsJsonArray("records");
        Assert.assertEquals(3, calcs.size());
        for (int i = 0; i < 3; i++) {
            var calc = calcs.get(i).getAsJsonObject();
            Assert.assertEquals("Element " + i + " should have exactly 1 key",
                1, calc.size());
            Assert.assertEquals("val_" + i + "_0",
                calc.get("s0").getAsString());
        }
    }

    @Test
    public void testTwoLevelDotNotationMultipleSubFields() throws Exception {
        // Keep 5 fields from each of 127 in the records
        setReturnObject(buildMapResponse(2));
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "data.records.s0", "data.records.d5",
                  "data.records.i10", "data.records.l15",
                  "data.records.b0"));

        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        JsonElement response = parseResponse();

        var calcs = response.getAsJsonObject().getAsJsonObject("data")
            .getAsJsonArray("records");
        var first = calcs.get(0).getAsJsonObject();
        Assert.assertEquals("Should have 5 keys per element", 5, first.size());
        Assert.assertTrue(first.has("s0"));
        Assert.assertTrue(first.has("d5"));
        Assert.assertTrue(first.has("i10"));
        Assert.assertTrue(first.has("l15"));
        Assert.assertTrue(first.has("b0"));
    }

    @Test
    public void testTwoLevelDotNotation97PercentReduction() throws Exception {
        // The dramatic test: 10 records x 127 fields, filter to 1
        MapResponse fullResp = buildMapResponse(10);

        // Full response
        setReturnObject(fullResp);
        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        int fullSize = outputStream.size();

        // Filtered: keep only data.records.s0
        outputStream.reset();
        outMsgContext.setProperty(JsonConstant.FIELD_FILTER,
            setOf("status", "data.records.s0"));
        formatter.writeTo(outMsgContext, outputFormat, outputStream, false);
        int filteredSize = outputStream.size();

        double reductionPct = (1.0 - (double) filteredSize / fullSize) * 100;

        Assert.assertTrue(
            "Full response (" + fullSize + " bytes) should be > 10KB",
            fullSize > 10000);
        Assert.assertTrue(
            "Filtered response (" + filteredSize + " bytes) should be < 500 bytes",
            filteredSize < 500);
        Assert.assertTrue(
            "Payload reduction (" + String.format("%.0f", reductionPct)
                + "%) should exceed 95%",
            reductionPct > 95.0);
    }

    static class TestHelper {
        static org.apache.axiom.om.OMElement createFaultElement() {
            var factory = OMAbstractFactory.getOMFactory();
            var ns = factory.createOMNamespace(
                "http://schemas.xmlsoap.org/soap/envelope/", "soapenv");
            var fault = factory.createOMElement("Fault", ns);
            var code = factory.createOMElement("faultcode", ns, fault);
            code.setText("soapenv:Server");
            var str = factory.createOMElement("faultstring", ns, fault);
            str.setText("test.Exception");
            var detail = factory.createOMElement("detail", ns, fault);
            detail.setText("filter test");
            return fault;
        }
    }
}
