[DEMO — do not merge] Green fprime-examples integration tests (full stack)

.github/workflows/build-test-rhel8.yml

@@ -32,7 +32,7 @@ jobs:
     steps:
     - name: "Install dependencies"
       run: |
-        dnf install -y git python3.12 python3.12-pip llvm-toolset libasan libubsan java-1.8.0-openjdk
+        dnf install -y git python3.12 python3.12-pip llvm-toolset libasan libubsan java-1.8.0-openjdk zlib-devel
         git config --global --add safe.directory ${GITHUB_WORKSPACE}
     - name: "Checkout F´ Repository"
       uses: actions/checkout@v4
@@ -52,7 +52,7 @@ jobs:
     steps:
     - name: "Install dependencies"
       run: |
-        dnf install -y git python3.12 python3.12-pip llvm-toolset libasan libubsan java-1.8.0-openjdk
+        dnf install -y git python3.12 python3.12-pip llvm-toolset libasan libubsan java-1.8.0-openjdk zlib-devel
         git config --global --add safe.directory ${GITHUB_WORKSPACE}
     - name: "Checkout F´ Repository"
       uses: actions/checkout@v4
@@ -73,7 +73,7 @@ jobs:
     steps:
     - name: "Install dependencies"
       run: |
-        dnf install -y git python3.12 python3.12-pip llvm-toolset libasan libubsan java-1.8.0-openjdk
+        dnf install -y git python3.12 python3.12-pip llvm-toolset libasan libubsan java-1.8.0-openjdk zlib-devel
         git config --global --add safe.directory ${GITHUB_WORKSPACE}
     - name: "Checkout F´ Repository"
       uses: actions/checkout@v4

.github/workflows/ext-build-examples-repo.yml

@@ -35,8 +35,18 @@ jobs:
     name: ""
     uses: ./.github/workflows/reusable-project-builder.yml
     with: 
-      target_repository: nasa/fprime-examples
+      # FORK-DEMO OVERRIDE: point at JPL-Devin/fprime-examples PR #1 so the new
+      # DataProduct/test/int compression test is present. Revert to
+      # `nasa/fprime-examples` + `needs.get-branch.outputs.target-branch` before
+      # opening this upstream.
+      target_repository: JPL-Devin/fprime-examples
       build_location: FlightExamples
       run_unit_tests: true
-      target_ref: ${{ needs.get-branch.outputs.target-branch }}
+      run_integration_tests: true
+      integration_gds_working_directory: FlightExamples/ExamplesDeployment
+      integration_test_working_directory: FlightExamples
+      integration_binary: ../build-artifacts/*/ExamplesDeployment/bin/ExamplesDeployment
+      integration_gds_args: "--dictionary ../build-artifacts/*/ExamplesDeployment/dict/ExamplesDeploymentTopologyDictionary.json"
+      integration_pytest_args: "DataProduct/test/int ManagerWorker/Subtopology/test/int --dictionary build-artifacts/*/ExamplesDeployment/dict/ExamplesDeploymentTopologyDictionary.json --no-zmq"
+      target_ref: devin/1783371774-dp-compression-example
       fprime_location: ./FlightExamples/lib/fprime

.github/workflows/reusable-project-builder.yml

@@ -19,6 +19,41 @@ on:
         required: false
         type: boolean
         default: true
+      run_integration_tests:
+        description: "Run an additional job to build the deployment and run integration tests against it."
+        required: false
+        type: boolean
+        default: false
+      integration_gds_working_directory:
+        description: "Directory to start fprime-gds from (relative to repo root). Required when run_integration_tests is true."
+        required: false
+        type: string
+        default: ""
+      integration_test_working_directory:
+        description: "Directory to run pytest from (relative to repo root). Required when run_integration_tests is true."
+        required: false
+        type: string
+        default: ""
+      integration_binary:
+        description: "Path to the FSW binary to run (relative to integration_gds_working_directory). Required when run_integration_tests is true."
+        required: false
+        type: string
+        default: ""
+      integration_binary_args:
+        description: "Arguments passed to the FSW binary."
+        required: false
+        type: string
+        default: "-a 127.0.0.1 -p 50000"
+      integration_gds_args:
+        description: "Extra arguments passed to fprime-gds (e.g. --dictionary path)."
+        required: false
+        type: string
+        default: ""
+      integration_pytest_args:
+        description: "Extra arguments passed to pytest (e.g. test paths, --dictionary path)."
+        required: false
+        type: string
+        default: ""
       fprime_location:
         description: "Relative path from the external project root to its F´ submodule"
         required: false
@@ -139,3 +174,77 @@ jobs:
           fprime-util check -j8 ${VERBOSE_FLAG} ${TARGET_PLATFORM} --pass-through --output-on-failure
         shell: bash
 
+  runIntegration:
+    if: ${{ inputs.run_integration_tests }}
+    runs-on: ${{ inputs.runs_on }}
+    name: "Integration Tests"
+    env:
+      FPRIME_LOCATION: ${{ inputs.fprime_location }}
+      TARGET_PLATFORM: ${{ inputs.target_platform }}
+    steps:
+      - name: "Checkout target repository"
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          repository: ${{ inputs.target_repository }}
+          ref: ${{ inputs.target_ref }}
+      - name: "Overlay current F´ revision"
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          path: ${{ inputs.fprime_location }}
+      # Install the target project's requirements so its GDS plugins (e.g. custom framing)
+      # are available to fprime-gds; fall back to the F´ submodule requirements otherwise.
+      - name: "Install requirements.txt"
+        run: |
+          # Modern build tooling is required so local source packages (e.g. the
+          # project's GDS plugins) build with correct metadata; the runner's
+          # system pip/setuptools can otherwise produce a broken "UNKNOWN" wheel
+          # with no entry points, which silently drops custom framing plugins.
+          pip3 install --upgrade pip setuptools wheel
+          if [ -f "${{ inputs.build_location }}/requirements.txt" ]; then
+            (cd "${{ inputs.build_location }}" && pip3 install -r requirements.txt)
+          else
+            pip3 install -r ${FPRIME_LOCATION}/requirements.txt
+          fi
+        shell: bash
+      # Match the F´ Ref integration tests: run against the in-development GDS
+      # (this PR's branch or devel) so GDS regressions are caught here too.
+      - name: "Upgrade fprime-gds to devel or pr-xxxx"
+        uses: nasa/fprime-actions/upgrade-package@devel
+        with:
+          repository: nasa/fprime-gds
+      - name: "Generate build cache"
+        working-directory: ${{ inputs.build_location }}
+        run: |
+          if [ "${RUNNER_DEBUG:-0}" = "1" ]; then
+            VERBOSE_FLAG=--verbose
+          else
+            VERBOSE_FLAG=
+          fi
+          fprime-util generate ${VERBOSE_FLAG} ${TARGET_PLATFORM}
+        shell: bash
+      - name: "Build"
+        working-directory: ${{ inputs.build_location }}
+        run: |
+          if [ "${RUNNER_DEBUG:-0}" = "1" ]; then
+            VERBOSE_FLAG=--verbose
+          else
+            VERBOSE_FLAG=
+          fi
+          fprime-util build -j8 ${VERBOSE_FLAG} ${TARGET_PLATFORM}
+        shell: bash
+      - name: "Make FSW binary executable"
+        working-directory: ${{ inputs.integration_gds_working_directory }}
+        run: chmod +x ${{ inputs.integration_binary }}
+        shell: bash
+      - name: "Integration Tests"
+        uses: nasa/fprime-actions/run-integration-tests@devel
+        with:
+          gds-working-directory: ${{ inputs.integration_gds_working_directory }}
+          test-working-directory: ${{ inputs.integration_test_working_directory }}
+          binary: ${{ inputs.integration_binary }}
+          binary-args: ${{ inputs.integration_binary_args }}
+          gds-args: ${{ inputs.integration_gds_args }}
+          pytest-args: ${{ inputs.integration_pytest_args }}
+

Fw/Types/Serializable.cpp

@@ -252,7 +252,7 @@
     }
     FW_ASSERT(this->getBuffAddr());
     // copy buffer to our buffer
-    (void)memcpy(&this->getBuffAddr()[this->m_serLoc], buff, static_cast<size_t>(length));
+    (void)memmove(&this->getBuffAddr()[this->m_serLoc], buff, static_cast<size_t>(length));
     this->m_serLoc += static_cast<Serializable::SizeType>(length);
     this->m_deserLoc = 0;
 

Svc/CMakeLists.txt

@@ -78,3 +78,5 @@ add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/Ccsds/")
 add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/ComAggregator/")
 add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/ComRetry/")
 add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/PassThroughRouter/")
+add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/DpCompressProc/")
+add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/DpZLibCompressor/")

Svc/Ccsds/SpacePacketDeframer/SpacePacketDeframer.cpp

@@ -20,6 +20,16 @@
 
 SpacePacketDeframer ::~SpacePacketDeframer() {}
 
+namespace {
+
+bool isValidPacketVersionNumber(const SpacePacketHeader& header) {
+    const U16 packetIdentification = header.get_packetIdentification();
+    const U16 pvn = (packetIdentification & SpacePacketSubfields::PvnMask) >> SpacePacketSubfields::PvnOffset;
+    return pvn == static_cast<U16>(ComCfg::Pvn::SPACE_PACKET_PROTOCOL);
+}
+
+}  // namespace
+
 // ----------------------------------------------------------------------
 // Handler implementations for typed input ports
 // ----------------------------------------------------------------------
@@ -35,7 +45,7 @@
     //  1b - 0/1 - (PT) Packet Type
     //  1b - 0/1 - (SHF) Secondary Header Flag
     // 11b - n/a - (APID) Application Process ID
-    //  2b - 00  - Sequence Flag
+    //  2b - 00/01/10/11 - Sequence Flag
     // 14b - n/a - Sequence Count
     // 16b - n/a - Packet Data Length
     // ################################
@@ -48,7 +58,7 @@
         }
         this->dataReturnOut_out(0, data, context);  // Drop the packet
         return;
     }
 
     SpacePacketHeader header;
     Fw::SerializeStatus status = data.getDeserializer().deserializeTo(header);
@@ -62,6 +72,15 @@
         return;
     }
 
+    if (!isValidPacketVersionNumber(header)) {
+        this->log_WARNING_HI_InvalidPacket();
+        if (this->isConnected_errorNotify_OutputPort(0)) {
+            this->errorNotify_out(0, Svc::Ccsds::FrameError::SP_INVALID_PACKET);
+        }
+        this->dataReturnOut_out(0, data, context);  // Drop the packet
+        return;
+    }
+
     // Widen to U32 before adding 1 to prevent U16 truncation to 0 when packetDataLength == 0xFFFF (max U16 value).
     // This is a undefined behavior condition in C++.
     const U32 pkt_length = static_cast<U32>(header.get_packetDataLength()) + 1U;

Svc/Ccsds/SpacePacketDeframer/docs/sdd.md

@@ -32,6 +32,7 @@ The `Svc::Ccsds::SpacePacketDeframer` extracts the following fields from the CCS
 | Name | Severity | Description |
 |---|---|---|
 | InvalidLength | `warning high` | Deframing received an invalid packet length |
+| InvalidPacket | `warning high` | Deframing received a malformed packet |
 
 ## Requirements
 
@@ -40,6 +41,7 @@ The `Svc::Ccsds::SpacePacketDeframer` extracts the following fields from the CCS
 | SVC-CCSDS-SPD-001 | The SpacePacketDeframer shall deframe CCSDS Space Packets according to the CCSDS 133.0-B-2 standard. | Unit Test, Inspection |
 | SVC-CCSDS-SPD-002 | The SpacePacketDeframer shall extract the user data field from valid Space Packets. | Unit Test |
 | SVC-CCSDS-SPD-003 | The SpacePacketDeframer shall validate the packet length of a Space Packet Primary Header. | Unit Test |
+| SVC-CCSDS-SPD-010 | The SpacePacketDeframer shall validate the Packet Version Number field and reject packets whose Packet Version Number does not match the CCSDS Space Packet Protocol, emitting `InvalidPacket` and returning the buffer. | Unit Test |
 | SVC-CCSDS-SPD-009 | The SpacePacketDeframer shall emit an `InvalidLength` event if the packet length token in the header is cannot fit in the received data, and drop the received packet. | Unit Test |
 | SVC-CCSDS-SPD-004 | The SpacePacketDeframer shall receive incoming data containing Space Packets via the `dataIn` port. | Unit Test |
 | SVC-CCSDS-SPD-005 | The SpacePacketDeframer shall output the extracted Space Packet user data via the `dataOut` port. | Unit Test |

Svc/Ccsds/SpacePacketDeframer/test/ut/SpacePacketDeframerTestMain.cpp

@@ -45,6 +45,26 @@ TEST(SpacePacketDeframer, testBufferSingleByte) {
     tester.testBufferSingleByte();
 }
 
+TEST(SpacePacketDeframer, testInvalidPacketIdentificationControlFields) {
+    Svc::Ccsds::SpacePacketDeframerTester tester;
+    tester.testInvalidPacketIdentificationControlFields();
+}
+
+TEST(SpacePacketDeframer, testCommandPacketTypeAccepted) {
+    Svc::Ccsds::SpacePacketDeframerTester tester;
+    tester.testCommandPacketTypeAccepted();
+}
+
+TEST(SpacePacketDeframer, testSecondaryHeaderFlagAccepted) {
+    Svc::Ccsds::SpacePacketDeframerTester tester;
+    tester.testSecondaryHeaderFlagAccepted();
+}
+
+TEST(SpacePacketDeframer, testSequenceFlagsAccepted) {
+    Svc::Ccsds::SpacePacketDeframerTester tester;
+    tester.testSequenceFlagsAccepted();
+}
+
 int main(int argc, char** argv) {
     ::testing::InitGoogleTest(&argc, argv);
     return RUN_ALL_TESTS();

Svc/Ccsds/SpacePacketDeframer/test/ut/SpacePacketDeframerTester.cpp

@@ -190,23 +190,97 @@ void SpacePacketDeframerTester ::testBufferSingleByte() {
     ASSERT_EVENTS_InvalidPacket_SIZE(1);
 }
 
+void SpacePacketDeframerTester ::testInvalidPacketIdentificationControlFields() {
+    U8 payload[2] = {0xAA, 0xBB};
+    ComCfg::FrameContext nullContext;
+
+    Fw::Buffer buffer =
+        this->assemblePacketWithControlFields(0x1,  // invalid PVN for Space Packet Protocol in this component
+                                              0x0, 0x0, static_cast<U16>(ComCfg::Apid::FW_PACKET_TELEM), 0x3, 0x0012,
+                                              static_cast<U16>(sizeof(payload) - 1), payload, sizeof(payload));
+
+    this->invoke_to_dataIn(0, buffer, nullContext);
+
+    ASSERT_from_dataOut_SIZE(0);
+    ASSERT_from_validateApidSeqCount_SIZE(0);
+    ASSERT_from_dataReturnOut_SIZE(1);
+    ASSERT_from_errorNotify(0, Svc::Ccsds::FrameError::SP_INVALID_PACKET);
+    ASSERT_FROM_PORT_HISTORY_SIZE(2);
+    ASSERT_EVENTS_SIZE(1);
+    ASSERT_EVENTS_InvalidPacket_SIZE(1);
+}
+
+void SpacePacketDeframerTester ::testCommandPacketTypeAccepted() {
+    this->testControlFieldAccepted(0x0, 0x1, 0x0, ComCfg::Apid::FW_PACKET_COMMAND, 0x3);
+}
+
+void SpacePacketDeframerTester ::testSecondaryHeaderFlagAccepted() {
+    this->testControlFieldAccepted(0x0, 0x0, 0x1, ComCfg::Apid::FW_PACKET_COMMAND, 0x3);
+}
+
+void SpacePacketDeframerTester ::testSequenceFlagsAccepted() {
+    this->testControlFieldAccepted(0x0, 0x0, 0x0, ComCfg::Apid::FW_PACKET_TELEM, 0x1);
+}
+
 // ----------------------------------------------------------------------
 // Helper functions
 // ----------------------------------------------------------------------
 
+void SpacePacketDeframerTester ::testControlFieldAccepted(U16 pvn,
+                                                          U16 packetType,
+                                                          U16 secondaryHeaderFlag,
+                                                          ComCfg::Apid::T expectedApid,
+                                                          U16 sequenceFlags) {
+    U8 payload[2] = {0xAA, 0xBB};
+    const U16 seqCount = 0x0012;
+    ComCfg::FrameContext nullContext;
+
+    Fw::Buffer buffer = this->assemblePacketWithControlFields(
+        pvn, packetType, secondaryHeaderFlag, static_cast<U16>(expectedApid), sequenceFlags, seqCount,
+        static_cast<U16>(sizeof(payload) - 1), payload, sizeof(payload));
+
+    this->invoke_to_dataIn(0, buffer, nullContext);
+
+    ASSERT_from_dataOut_SIZE(1);
+    ASSERT_from_validateApidSeqCount_SIZE(1);
+    ASSERT_from_dataReturnOut_SIZE(0);
+    ASSERT_FROM_PORT_HISTORY_SIZE(2);
+    ASSERT_EQ(this->fromPortHistory_dataOut->at(0).context.get_apid(), expectedApid);
+    ASSERT_EQ(this->fromPortHistory_dataOut->at(0).context.get_sequenceCount(), seqCount);
+    ASSERT_EVENTS_SIZE(0);
+}
+
 Fw::Buffer SpacePacketDeframerTester ::assemblePacket(U16 apid,
                                                       U16 seqCount,
                                                       U16 lengthToken,
                                                       U8* packetData,
                                                       U16 packetDataLen,
                                                       bool hasSecHdr) {
+    return this->assemblePacketWithControlFields(0x0, 0x0, static_cast<U16>(hasSecHdr), apid, 0x3, seqCount,
+                                                 lengthToken, packetData, packetDataLen);
+}
+
+Fw::Buffer SpacePacketDeframerTester ::assemblePacketWithControlFields(U16 pvn,
+                                                                       U16 packetType,
+                                                                       U16 secondaryHeaderFlag,
+                                                                       U16 apid,
+                                                                       U16 sequenceFlags,
+                                                                       U16 seqCount,
+                                                                       U16 lengthToken,
+                                                                       U8* packetData,
+                                                                       U16 packetDataLen) {
     SpacePacketHeader header;
-    U16 packetId = apid & SpacePacketSubfields::ApidMask;
-    if (hasSecHdr) {
-        packetId |= SpacePacketSubfields::SecHdrMask;
-    }
-    header.set_packetIdentification(packetId);
-    header.set_packetSequenceControl(seqCount);  // Sequence Flags = 0b11 (unsegmented) & unused Seq count
+    const U16 packetIdentification = static_cast<U16>(
+        ((pvn << SpacePacketSubfields::PvnOffset) & SpacePacketSubfields::PvnMask) |
+        ((packetType << SpacePacketSubfields::PktTypeOffset) & SpacePacketSubfields::PktTypeMask) |
+        ((secondaryHeaderFlag << SpacePacketSubfields::SecHdrOffset) & SpacePacketSubfields::SecHdrMask) |
+        (apid & SpacePacketSubfields::ApidMask));
+    const U16 packetSequenceControl = static_cast<U16>(
+        ((sequenceFlags << SpacePacketSubfields::SeqFlagsOffset) & SpacePacketSubfields::SeqFlagsMask) |
+        (seqCount & SpacePacketSubfields::SeqCountMask));
+
+    header.set_packetIdentification(packetIdentification);
+    header.set_packetSequenceControl(packetSequenceControl);
     header.set_packetDataLength(lengthToken);
 
     Fw::ExternalSerializeBuffer serializer(static_cast<U8*>(this->m_packetBuffer), sizeof(this->m_packetBuffer));