From 02f1b4592e10c0033bb27b5c78e06b5fbea22b9d Mon Sep 17 00:00:00 2001 From: Max Brombach Date: Wed, 15 Nov 2023 16:02:47 +0000 Subject: [PATCH] Device/Disk: Add VSA read and parse functionality Implement ability to extract network traffic (CAN, LIN, Ethernet, etc.) from VSA message records on disk. Add a method to Device class that uses the VSAParser and the individual record types to extract messages from the VSA message records and pass them back to the communication system. This routes messages such that it appears as if they were discovered live instead of read from disk. The parse process (in Device) requires determination of metadata about the VSA file system on a device before it can begin extracting messages. This currently only handles data captured from the current coremini script on a device. --- CMakeLists.txt | 16 + api/icsneocpp/event.cpp | 25 + device/device.cpp | 1110 ++++++++++++++++- disk/vsa/vsa.cpp | 39 + disk/vsa/vsa02.cpp | 26 + disk/vsa/vsa03.cpp | 24 + disk/vsa/vsa04.cpp | 24 + disk/vsa/vsa05.cpp | 23 + disk/vsa/vsa06.cpp | 25 + disk/vsa/vsa07.cpp | 25 + disk/vsa/vsa08.cpp | 24 + disk/vsa/vsa09.cpp | 32 + disk/vsa/vsa0b.cpp | 38 + disk/vsa/vsa0c.cpp | 27 + disk/vsa/vsa0d.cpp | 104 ++ disk/vsa/vsa0e.cpp | 111 ++ disk/vsa/vsa0f.cpp | 106 ++ disk/vsa/vsa6a.cpp | 37 + disk/vsa/vsaparser.cpp | 460 +++++++ examples/CMakeLists.txt | 5 + examples/cpp/vsa/CMakeLists.txt | 27 + examples/cpp/vsa/src/VSAExample.cpp | 94 ++ include/icsneo/api/event.h | 9 + include/icsneo/communication/communication.h | 3 +- .../message/filter/messagefilter.h | 6 +- include/icsneo/device/device.h | 187 +++ .../device/tree/radgigastar/radgigastar.h | 2 +- include/icsneo/disk/vsa/vsa.h | 312 +++++ include/icsneo/disk/vsa/vsa02.h | 55 + include/icsneo/disk/vsa/vsa03.h | 54 + include/icsneo/disk/vsa/vsa04.h | 49 + include/icsneo/disk/vsa/vsa05.h | 79 ++ include/icsneo/disk/vsa/vsa06.h | 49 + include/icsneo/disk/vsa/vsa07.h | 49 + include/icsneo/disk/vsa/vsa08.h | 46 + include/icsneo/disk/vsa/vsa09.h | 63 + include/icsneo/disk/vsa/vsa0b.h | 57 + include/icsneo/disk/vsa/vsa0c.h | 54 + include/icsneo/disk/vsa/vsa0d.h | 134 ++ include/icsneo/disk/vsa/vsa0e.h | 130 ++ include/icsneo/disk/vsa/vsa0f.h | 119 ++ include/icsneo/disk/vsa/vsa6a.h | 49 + include/icsneo/disk/vsa/vsaparser.h | 234 ++++ 43 files changed, 4122 insertions(+), 20 deletions(-) create mode 100644 disk/vsa/vsa.cpp create mode 100644 disk/vsa/vsa02.cpp create mode 100644 disk/vsa/vsa03.cpp create mode 100644 disk/vsa/vsa04.cpp create mode 100644 disk/vsa/vsa05.cpp create mode 100644 disk/vsa/vsa06.cpp create mode 100644 disk/vsa/vsa07.cpp create mode 100644 disk/vsa/vsa08.cpp create mode 100644 disk/vsa/vsa09.cpp create mode 100644 disk/vsa/vsa0b.cpp create mode 100644 disk/vsa/vsa0c.cpp create mode 100644 disk/vsa/vsa0d.cpp create mode 100644 disk/vsa/vsa0e.cpp create mode 100644 disk/vsa/vsa0f.cpp create mode 100644 disk/vsa/vsa6a.cpp create mode 100644 disk/vsa/vsaparser.cpp create mode 100644 examples/cpp/vsa/CMakeLists.txt create mode 100644 examples/cpp/vsa/src/VSAExample.cpp create mode 100644 include/icsneo/disk/vsa/vsa.h create mode 100644 include/icsneo/disk/vsa/vsa02.h create mode 100644 include/icsneo/disk/vsa/vsa03.h create mode 100644 include/icsneo/disk/vsa/vsa04.h create mode 100644 include/icsneo/disk/vsa/vsa05.h create mode 100644 include/icsneo/disk/vsa/vsa06.h create mode 100644 include/icsneo/disk/vsa/vsa07.h create mode 100644 include/icsneo/disk/vsa/vsa08.h create mode 100644 include/icsneo/disk/vsa/vsa09.h create mode 100644 include/icsneo/disk/vsa/vsa0b.h create mode 100644 include/icsneo/disk/vsa/vsa0c.h create mode 100644 include/icsneo/disk/vsa/vsa0d.h create mode 100644 include/icsneo/disk/vsa/vsa0e.h create mode 100644 include/icsneo/disk/vsa/vsa0f.h create mode 100644 include/icsneo/disk/vsa/vsa6a.h create mode 100644 include/icsneo/disk/vsa/vsaparser.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d3bd00..1281009 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -275,6 +275,22 @@ set(SRC_FILES disk/plasiondiskreaddriver.cpp disk/extextractordiskreaddriver.cpp disk/fat.cpp + disk/vsa/vsa.cpp + disk/vsa/vsa02.cpp + disk/vsa/vsa03.cpp + disk/vsa/vsa04.cpp + disk/vsa/vsa05.cpp + disk/vsa/vsa06.cpp + disk/vsa/vsa07.cpp + disk/vsa/vsa08.cpp + disk/vsa/vsa09.cpp + disk/vsa/vsa0b.cpp + disk/vsa/vsa0c.cpp + disk/vsa/vsa0d.cpp + disk/vsa/vsa0e.cpp + disk/vsa/vsa0f.cpp + disk/vsa/vsa6a.cpp + disk/vsa/vsaparser.cpp ${PLATFORM_SRC} ) diff --git a/api/icsneocpp/event.cpp b/api/icsneocpp/event.cpp index 11aec0b..d9dc245 100644 --- a/api/icsneocpp/event.cpp +++ b/api/icsneocpp/event.cpp @@ -169,6 +169,15 @@ static constexpr const char* FT_DEVICE_NOT_CONNECTED = "FTD3XX device not connec static constexpr const char* FT_INCORRECT_DEVICE_PATH = "Incorrect FTD3XX device path."; static constexpr const char* FT_OTHER_ERROR = "Other FTD3XX error."; +// VSA +static constexpr const char* VSA_BUFFER_CORRUPTED = "VSA data in record buffer is corrupted."; +static constexpr const char* VSA_TIMESTAMP_NOT_FOUND = "Unable to find a VSA record with a valid timestamp."; +static constexpr const char* VSA_BUFFER_FORMAT_ERROR = "VSA record buffer is formatted incorrectly."; +static constexpr const char* VSA_MAX_READ_ATTEMPTS_REACHED = "Reached max attempts to read VSA records before exit."; +static constexpr const char* VSA_BYTE_PARSE_FAILURE = "Failure to parse record bytes from VSA buffer."; +static constexpr const char* VSA_EXTENDED_MESSAGE_ERROR = "Failure to parse extended message record sequence"; +static constexpr const char* VSA_OTHER_ERROR = "Unknown error in VSA read API."; + static constexpr const char* TOO_MANY_EVENTS = "Too many events have occurred. The list has been truncated."; static constexpr const char* UNKNOWN = "An unknown internal error occurred."; static constexpr const char* INVALID = "An invalid internal error occurred."; @@ -393,6 +402,22 @@ const char* APIEvent::DescriptionForType(Type type) { case Type::FTOtherError: return FT_OTHER_ERROR; + // VSA + case Type::VSABufferCorrupted: + return VSA_BUFFER_CORRUPTED; + case Type::VSATimestampNotFound: + return VSA_TIMESTAMP_NOT_FOUND; + case Type::VSABufferFormatError: + return VSA_BUFFER_FORMAT_ERROR; + case Type::VSAMaxReadAttemptsReached: + return VSA_MAX_READ_ATTEMPTS_REACHED; + case Type::VSAByteParseFailure: + return VSA_BYTE_PARSE_FAILURE; + case Type::VSAExtendedMessageError: + return VSA_EXTENDED_MESSAGE_ERROR; + case Type::VSAOtherError: + return VSA_OTHER_ERROR; + // Other Errors case Type::TooManyEvents: return TOO_MANY_EVENTS; diff --git a/device/device.cpp b/device/device.cpp index 7c8e508..c0e566b 100644 --- a/device/device.cpp +++ b/device/device.cpp @@ -480,10 +480,15 @@ bool Device::startScript(Disk::MemoryType memType) } uint8_t location = static_cast(memType); - auto generic = com->sendCommand(Command::LoadCoreMini, location); - if(!generic) - { + std::shared_ptr filter = std::make_shared(Network::NetID::Device); + filter->includeInternalInAny = true; + + const auto response = com->waitForMessageSync([&]() { + return com->sendCommand(Command::LoadCoreMini, location); + }, filter); + + if(!response) { report(APIEvent::Type::NoDeviceResponse, APIEvent::Severity::Error); return false; } @@ -498,10 +503,14 @@ bool Device::stopScript() return false; } - auto generic = com->sendCommand(Command::ClearCoreMini); + std::shared_ptr filter = std::make_shared(Network::NetID::Device); + filter->includeInternalInAny = true; - if(!generic) - { + const auto response = com->waitForMessageSync([&]() { + return com->sendCommand(Command::ClearCoreMini); + }, filter); + + if(!response) { report(APIEvent::Type::NoDeviceResponse, APIEvent::Severity::Error); return false; } @@ -753,7 +762,7 @@ std::optional Device::isLogicalDiskConnected() { } const auto info = com->getLogicalDiskInfoSync(); - if (!info) { + if(!info) { report(APIEvent::Type::Timeout, APIEvent::Severity::Error); return std::nullopt; } @@ -768,14 +777,14 @@ std::optional Device::getLogicalDiskSize() { } const auto info = com->getLogicalDiskInfoSync(); - if (!info) { + if(!info) { report(APIEvent::Type::Timeout, APIEvent::Severity::Error); return std::nullopt; } const auto reportedSize = info->getReportedSize(); - if (diskReadDriver->getAccess() == Disk::Access::VSA) + if(diskReadDriver->getAccess() == Disk::Access::VSA) return reportedSize - diskReadDriver->getVSAOffset(); return reportedSize; @@ -787,7 +796,7 @@ std::optional Device::getVSAOffsetInLogicalDisk() { return std::nullopt; } - if (diskReadDriver->getAccess() == Disk::Access::VSA || diskReadDriver->getAccess() == Disk::Access::None) + if(diskReadDriver->getAccess() == Disk::Access::VSA || diskReadDriver->getAccess() == Disk::Access::None) return 0ull; auto offset = Disk::FindVSAInFAT([this](uint64_t pos, uint8_t *into, uint64_t amount) { @@ -1324,7 +1333,7 @@ void Device::scriptStatusThreadBody() const auto resp = getScriptStatus(); - if (!resp) + if(!resp) continue; //If value changed/was inserted, notify callback @@ -1702,7 +1711,7 @@ std::optional Device::sendEthPhyMsg(const EthPhyMessage& message, std::optional Device::SetCollectionUploaded(uint32_t collectionEntryByteAddress) { - if (!supportsWiVI()) + if(!supportsWiVI()) { report(APIEvent::Type::WiVINotSupported, APIEvent::Severity::EventWarning); return std::nullopt; @@ -1718,20 +1727,20 @@ std::optional Device::SetCollectionUploaded(uint32_t collectionEntryByteAd std::shared_ptr response = com->waitForMessageSync( [this, args](){ return com->sendCommand(ExtendedCommand::SetUploadedFlag, args); }, std::make_shared(Message::Type::ExtendedResponse), timeout); - if (!response) + if(!response) { report(APIEvent::Type::NoDeviceResponse, APIEvent::Severity::Error); return std::nullopt; } auto retMsg = std::static_pointer_cast(response); - if (!retMsg) + if(!retMsg) { // TODO fix this error report(APIEvent::Type::NoDeviceResponse, APIEvent::Severity::Error); return std::make_optional(false); } bool success = retMsg->response == ExtendedResponse::OK; - if (!success) + if(!success) { // TODO fix this error report(APIEvent::Type::Unknown, APIEvent::Severity::EventWarning); @@ -2043,4 +2052,1075 @@ bool Device::clearAllLiveData() { } return true; +} + +bool Device::readVSA(const VSAExtractionSettings& extractionSettings) { + if(isOnline()) { + goOffline(); + } + auto innerReadVSA = [&](uint64_t diskSize) -> const bool { + // Adjust driver to offset to start of VSA file + const auto& offset = getVSAOffsetInLogicalDisk(); + if(!offset) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + diskReadDriver->setVSAOffset(*offset); + + // Gather metadata about VSA file system + VSAMetadata metadata; + + metadata.diskSize = diskSize; + + if(!probeVSA(metadata, extractionSettings)) { + return false; + } + + if(extractionSettings.filters.empty()) { // Full SD Dump + if(!parseVSA(metadata, extractionSettings)) { + return false; + } + } else { // Only read data specified by filters + for(const auto& filter : extractionSettings.filters) { + if(!parseVSA(metadata, extractionSettings, filter)) { + return false; + } + } + } + return true; + }; + + auto bufferSize = getVSADiskSize(); + + if(!bufferSize) { + return false; + } + + const auto& wasScriptStarted = getScriptStatus()->isCoreminiRunning; + + if(extractionSettings.stopCoreMini && wasScriptStarted) { + stopScript(); + } + + const auto ret = innerReadVSA(*bufferSize); + + // Restart CoreMini script if stopped + if(extractionSettings.stopCoreMini && wasScriptStarted) { + startScript(); + } + return ret; +} + +bool Device::probeVSA(VSAMetadata& metadata, const VSAExtractionSettings& extractionSettings) { + auto cmTimestamp = getCoreMiniScriptTimestamp(); + if(!metadata.coreMiniTimestamp) { + return false; + } + metadata.coreMiniTimestamp = *cmTimestamp; + + const auto& isOverlapped = isVSAOverlapped(metadata); + if(isOverlapped) { + metadata.isOverlapped = *isOverlapped; + } else { + return false; + } + + if(!findFirstVSARecord(metadata.firstRecordLocation, metadata.firstRecord, extractionSettings, metadata)) { + return false; + } + + if(!findLastVSARecord(metadata.lastRecordLocation, metadata.lastRecord, extractionSettings, metadata)) { + return false; + } + + if(metadata.isOverlapped) { + // The last byte in the buffer should immediately precede the first if the buffer is overlapped + metadata.bufferEnd = metadata.firstRecordLocation; + } else { + metadata.bufferEnd = metadata.lastRecordLocation; + const auto& type = metadata.lastRecord->getType(); + if(type == VSA::Type::AA0D || type == VSA::Type::AA0E || type == VSA::Type::AA0F) { + // Add bytes based off of how many records should be in extended record sequence + metadata.bufferEnd += std::dynamic_pointer_cast(metadata.lastRecord)->getRecordCount() * VSA::StandardRecordSize; + } else if(type == VSA::Type::AA6A) { + // Add a full sector for script status backup records + metadata.bufferEnd += Disk::SectorSize; + } else { + // All other records add only one single record offset. + metadata.bufferEnd += VSA::StandardRecordSize; + } + } + return true; +} + +bool Device::findFirstVSARecord(uint64_t& firstOffset, std::shared_ptr& firstRecord, + const VSAExtractionSettings& extractionSettings, std::optional optMetadata) { + // Grab important metadata features if metadata not defined + VSAMetadata metadata; + if(!optMetadata) { + const auto& coreMiniTimestamp = getCoreMiniScriptTimestamp(); + if(!coreMiniTimestamp) { + return false; + } + metadata.coreMiniTimestamp = *coreMiniTimestamp; + const auto& diskSize = getVSADiskSize(); + if(!diskSize) { + return false; + } + metadata.diskSize = *diskSize; + const auto& isOverlapped = isVSAOverlapped(metadata); + if(isOverlapped) { + metadata.isOverlapped = *isOverlapped; + } else { + return false; + } + } else { + metadata = *optMetadata; + } + + if(!metadata.isOverlapped) { // Grab the first record in the buffer + std::vector buffer; + buffer.resize(Disk::SectorSize); + const auto& bytesRead = readLogicalDisk(VSA::RecordStartOffset, buffer.data(), Disk::SectorSize); + if(!bytesRead || *bytesRead < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + VSAParser parser(report); + std::shared_ptr record; + const auto& firstRecordStatus = parser.getRecordFromBytes(buffer.data(), Disk::SectorSize, record); + if(firstRecordStatus == VSAParser::RecordParseStatus::Success) { + firstOffset = VSA::RecordStartOffset; + firstRecord = record; + return true; + } + report(APIEvent::Type::VSAOtherError, APIEvent::Severity::Error); + return false; + } + + static constexpr size_t RegionCount = 2; // Number of regions to chunk remaining disk into during logarithmic search + static constexpr size_t LinearSearchMaxSize = 0x00010000u; // Size of remaining disk space at which to begin iterative search (64 KB) + + // Initialize variables for disk search + VSAParser parser(report); + uint64_t first = VSA::RecordStartOffset; // First byte of vsa records to search + uint64_t last = metadata.diskSize; // One beyond the last byte of vsa records to search + uint64_t lastSector = last - Disk::SectorSize; + std::vector buffer; + buffer.resize(Disk::SectorSize); + uint64_t start = first; // First byte of data to search for this iteration of algorithm + uint64_t stop = last; // One beyond last byte of data to search for this iteration of algorithm + + // Repeatedly chunk data into regions and find largest negative difference in timestamp + // (i.e., where the timestamp at the beginning of the region is much larger than at the end) + // until data is under the LinearSearchMaxSize + while(stop - start > LinearSearchMaxSize) { + uint64_t regionSize = (stop - start) / RegionCount; + regionSize -= regionSize % Disk::SectorSize; // Ensures regions are chunked on a sector boundary + uint64_t largestNegative = 0; + size_t largestNegativeIndex = 0; + uint64_t smallestStartTimestamp = UINT64_MAX; + size_t smallestStartTimestampIndex = 0; // Only necessary in cases where smallest timestamp falls on front edge of a region + for(size_t i = 0; i < RegionCount; i++) { + uint64_t regionStart = start + i * regionSize; + // Start of last sector to read in the current region + uint64_t regionLastSector = (i != RegionCount - 1) + ? regionStart + regionSize - Disk::SectorSize + : (stop - Disk::SectorSize) - ((stop - Disk::SectorSize) % Disk::SectorSize); + const auto& timestampStart = getVSATimestampOrBefore(parser, buffer, regionStart, first, metadata); + const auto& timestampStop = getVSATimestampOrAfter(parser, buffer, regionLastSector, lastSector, metadata); + // Ensure valid timestamps were found + if(!timestampStart || !timestampStop) { + return false; + } + // Check if region has largest negative + if(*timestampStart > *timestampStop && *timestampStart - *timestampStop > largestNegative) { + if(*timestampStop > metadata.coreMiniTimestamp || extractionSettings.parseOldRecords) { + largestNegative = *timestampStart - *timestampStop; + largestNegativeIndex = i; + } + } + // Track smallest start timestamp for edge case + if(*timestampStart < smallestStartTimestamp) { + if(*timestampStart >= metadata.coreMiniTimestamp || extractionSettings.parseOldRecords) { + smallestStartTimestamp = *timestampStart; + smallestStartTimestampIndex = i; + } + } + } + + if(largestNegative == 0) { + // We did not find a switch between large and small timestamps within a region. + // Therefore the smallest timestamp is the first record within one of the regions. + uint64_t location = start + smallestStartTimestampIndex * regionSize; + const auto& bytesRead = readLogicalDisk(location, buffer.data(), Disk::SectorSize); + if(!bytesRead || *bytesRead < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + std::shared_ptr record; + parser.getRecordFromBytes(buffer.data(), Disk::SectorSize, record); + if(record) { + firstRecord = record; + firstOffset = location; + return true; + } + report(APIEvent::Type::VSAByteParseFailure, APIEvent::Severity::Error); + return false; + } + + // Set start and stop to be start and stop of largest negative region + start += largestNegativeIndex * regionSize; + stop = (largestNegativeIndex != RegionCount - 1) ? start + regionSize : stop; + } + + // Initialize variables for iterative search + auto smallestTimestampLocation = UINT64_MAX; + auto smallestTimestamp = UINT64_MAX; + size_t regionSize = static_cast(stop - start - ((stop - start) % Disk::SectorSize)); + buffer.resize(regionSize); + const auto& bytesRead = readLogicalDisk(start, buffer.data(), regionSize); + if(!bytesRead || *bytesRead < regionSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + std::shared_ptr minRecord; + + // Iteratively find sector with smallest timestamp from largest negative region + for(size_t offset = 0; offset + VSA::StandardRecordSize < regionSize; offset += VSA::StandardRecordSize) { + std::shared_ptr record; + auto recordParseStatus = parser.getRecordFromBytes(buffer.data() + offset, Disk::SectorSize, record); + + if(recordParseStatus == VSAParser::RecordParseStatus::ConsecutiveExtended) { + // Backtrack to get timestamp if first record is consecutive extended record + auto extendedRecord = std::dynamic_pointer_cast(record); + auto pos = start + offset; + if(!findFirstExtendedVSAFromConsecutive(extendedRecord, pos, parser, metadata)) { + // Possibly erased by looped buffer + continue; + } + uint64_t timestamp = extendedRecord->getTimestamp(); + if(timestamp < smallestTimestamp) { + smallestTimestampLocation = pos; + smallestTimestamp = timestamp; + minRecord = extendedRecord; + } + } else if(record) { + // Update data tracking record with minimum timestamp + uint64_t timestamp = record->getTimestamp(); + if(timestamp < smallestTimestamp) { + smallestTimestampLocation = offset + start; + smallestTimestamp = timestamp; + minRecord = record; + } + } + } + if(smallestTimestamp == UINT64_MAX || !minRecord) { + report(APIEvent::Type::VSATimestampNotFound, APIEvent::Severity::Error); + return false; + } + firstOffset = smallestTimestampLocation; + firstRecord = minRecord; + return true; +} + +bool Device::findLastVSARecord(uint64_t& lastOffset, std::shared_ptr& lastRecord, + const VSAExtractionSettings& extractionSettings, std::optional optMetadata) { + static constexpr auto LinearSearchSize = 0x8000u; + + // Grab important metadata features if metadata not defined + VSAMetadata metadata; + if(optMetadata) { + metadata = *optMetadata; + } else { + const auto& coreMiniTimestamp = getCoreMiniScriptTimestamp(); + if(!coreMiniTimestamp) { + return false; + } + metadata.coreMiniTimestamp = *coreMiniTimestamp; + const auto& diskSize = getVSADiskSize(); + if(!diskSize) { + return false; + } + metadata.diskSize = *diskSize; + const auto& isOverlapped = isVSAOverlapped(metadata); + if(!isOverlapped) { + return false; + } + metadata.isOverlapped = *isOverlapped; + } + + // Find record prior to first (chronological) record if VSA buffer is overlapped + VSAParser parser(report); + if(metadata.isOverlapped) { + uint64_t firstOffset; + std::shared_ptr firstRecord; + if(metadata.firstRecordLocation != UINT64_MAX && metadata.firstRecord) { + firstOffset = metadata.firstRecordLocation; + } else if(!findFirstVSARecord(firstOffset, firstRecord, extractionSettings, metadata)) { + return false; + } + std::vector buffer; + buffer.resize(Disk::SectorSize); + // Read sector before first if buffer is looped + const auto& bytesRead = vsaReadLogicalDisk(firstOffset - Disk::SectorSize, buffer.data(), Disk::SectorSize, metadata); + if(!bytesRead || *bytesRead < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + uint16_t bufferOffset = Disk::SectorSize; + do { + bufferOffset -= VSA::StandardRecordSize; + const auto& lastRecordStatus = parser.getRecordFromBytes(buffer.data() + bufferOffset, VSA::StandardRecordSize, lastRecord); + if(lastRecordStatus == VSAParser::RecordParseStatus::Success) { + lastOffset = firstOffset - Disk::SectorSize + bufferOffset; + return true; + } else if(lastRecordStatus != VSAParser::RecordParseStatus::NotARecordStart) { + // Reverse search for record with valid timestamp + // Necessary if previous record is a pad record or consecutive extended record + auto pos = firstOffset - bufferOffset; + if(findPreviousRecordWithTimestamp(lastRecord, pos, parser)) { + lastOffset = pos; + return true; + } + return true; + } + } while(bufferOffset > 0); + lastRecord = nullptr; + report(APIEvent::Type::VSAOtherError, APIEvent::Severity::Error); + return false; + } + + // Binary search for last record until region is less than LinearSearchSize + std::vector buffer; + buffer.resize(Disk::SectorSize); + uint64_t left = VSA::RecordStartOffset; + uint64_t right = metadata.diskSize; + while(right - left > LinearSearchSize) { + uint64_t pos = (left + right) / 2; + pos -= pos % Disk::SectorSize; + const auto& bytesRead = readLogicalDisk(pos, buffer.data(), Disk::SectorSize); + if(!bytesRead || *bytesRead < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + std::shared_ptr testRecord; + const auto& testStatus = parser.getRecordFromBytes(buffer.data(), Disk::SectorSize, testRecord); + if(testStatus == VSAParser::RecordParseStatus::NotARecordStart) { + right = pos; + } else if(testStatus == VSAParser::RecordParseStatus::ConsecutiveExtended) { + auto extendedRecord = std::dynamic_pointer_cast(testRecord); + auto tempPos = pos; + if(!findFirstExtendedVSAFromConsecutive(extendedRecord, tempPos, parser, metadata)) { + tempPos = pos; + if(!findPreviousRecordWithTimestamp(testRecord, tempPos, parser)) { + return false; + } + if(testRecord->getTimestamp() > metadata.coreMiniTimestamp || extractionSettings.parseOldRecords) { + left = tempPos; + } else { + right = tempPos; + } + } + if(extendedRecord->getTimestamp() > metadata.coreMiniTimestamp || extractionSettings.parseOldRecords) { + left = tempPos; + } else { + right = tempPos; + } + } else { + if(testRecord->getTimestamp() > metadata.coreMiniTimestamp || extractionSettings.parseOldRecords) { + left = pos; + } else { + right = pos; + } + } + } + + buffer.resize(static_cast(LinearSearchSize)); + const auto& bytesRead = readLogicalDisk(left, buffer.data(), LinearSearchSize); + if(!bytesRead || *bytesRead < LinearSearchSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + std::shared_ptr previousRecord; + auto previousParseStatus = parser.getRecordFromBytes(buffer.data(), LinearSearchSize, previousRecord); + uint64_t previousParseLocation = left; + uint64_t bufferOffset = VSA::StandardRecordSize; + + // Ensure we have a record with a valid timestamp as the previousParseResult + while(previousParseStatus != VSAParser::RecordParseStatus::Success) { + if(bufferOffset >= LinearSearchSize) { // Never found a record with a valid timestamp (unlikely) + report(APIEvent::Type::VSAOtherError, APIEvent::Severity::Error); + return false; + } + previousParseStatus = parser.getRecordFromBytes(buffer.data() + bufferOffset, static_cast(LinearSearchSize - bufferOffset), previousRecord); + previousParseLocation = left + bufferOffset; + bufferOffset += VSA::StandardRecordSize; + } + + // Perform linear search for last record + while(bufferOffset < LinearSearchSize) { + std::shared_ptr record; + auto parseStatus = parser.getRecordFromBytes(buffer.data() + bufferOffset, static_cast(LinearSearchSize - bufferOffset), record); + if(parseStatus == VSAParser::RecordParseStatus::NotARecordStart) { + // We found the end of the VSA buffer + // Return the last valid record we found + lastOffset = previousParseLocation; + lastRecord = previousRecord; + return true; + } else if( + parseStatus == VSAParser::RecordParseStatus::Success && + record->getTimestamp() < metadata.coreMiniTimestamp && + !extractionSettings.parseOldRecords + ) { // We have entered outdated record data + lastOffset = previousParseLocation; + lastRecord = previousRecord; + return true; + } else if(parseStatus == VSAParser::RecordParseStatus::Success) { + // Save new last-record data and update bufferOffset according to record size + previousParseStatus = parseStatus; + previousRecord = record; + previousParseLocation = left + bufferOffset; + bufferOffset += (record->getType() == VSA::Type::AA6A) ? Disk::SectorSize : VSA::StandardRecordSize; + } else { + bufferOffset += VSA::StandardRecordSize; + } + } + report(APIEvent::Type::VSAOtherError, APIEvent::Severity::Error); + return false; // Somehow we did not find any non-record data despite non-overlapped buffer +} + +std::optional Device::isVSAOverlapped(std::optional optMetadata) { + // Grab important metadata features if metadata not defined + VSAMetadata metadata; + if(!optMetadata) { + const auto& diskSize = getVSADiskSize(); + if(!diskSize) { + return std::nullopt; + } + metadata.diskSize = *diskSize; + const auto& coreMiniTimestamp = getCoreMiniScriptTimestamp(); + if(!coreMiniTimestamp) { + return std::nullopt; + } + metadata.coreMiniTimestamp = *coreMiniTimestamp; + } else { + metadata = *optMetadata; + } + + // Read first sector + VSAParser parser(report); + std::vector buffer; + buffer.resize(Disk::SectorSize); + const auto& bytesReadFirst = readLogicalDisk(VSA::RecordStartOffset, buffer.data(), Disk::SectorSize); + if(!bytesReadFirst || *bytesReadFirst < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return std::nullopt; + } + std::shared_ptr firstRecord; + auto firstRecordStatus = parser.getRecordFromBytes(buffer.data(), Disk::SectorSize, firstRecord); + if(firstRecordStatus == VSAParser::RecordParseStatus::NotARecordStart) { + report(APIEvent::Type::VSABufferCorrupted, APIEvent::Severity::Error); + return std::nullopt; // Beginning of buffer is not a record + } else if(firstRecordStatus == VSAParser::RecordParseStatus::ConsecutiveExtended) { + return true; // The only way to have a consecutive extended record at the beginning is if the buffer looped + } + + // Read last sector + uint64_t lastPos = metadata.diskSize - Disk::SectorSize; + lastPos -= lastPos % Disk::SectorSize; + const auto& bytesReadLast = readLogicalDisk(lastPos, buffer.data(), Disk::SectorSize); + if(!bytesReadLast || *bytesReadLast < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return std::nullopt; + } + std::shared_ptr lastSectorRecord; + auto lastSectorStatus = parser.getRecordFromBytes(buffer.data(), Disk::SectorSize, lastSectorRecord); + if(lastSectorStatus == VSAParser::RecordParseStatus::ConsecutiveExtended) { + // Find the beginning record of the extended record sequence + auto extendedRecord = std::dynamic_pointer_cast(lastSectorRecord); + if(!findFirstExtendedVSAFromConsecutive(extendedRecord, lastPos, parser, metadata)) { + return std::nullopt; + } + return firstRecord->getTimestamp() >= extendedRecord->getTimestamp() && + extendedRecord->getTimestamp() > metadata.coreMiniTimestamp; + } else if(firstRecord && lastSectorRecord && firstRecord->isTimestampValid() && lastSectorRecord->isTimestampValid()) { + // Handle situation where both first and last records have valid timestamps + return firstRecord->getTimestamp() >= lastSectorRecord->getTimestamp() && + lastSectorRecord->getTimestamp() > metadata.coreMiniTimestamp; + } else if(lastSectorStatus == VSAParser::RecordParseStatus::NotARecordStart) { + // The vsa record buffer is not full + report(APIEvent::Type::VSAOtherError, APIEvent::Severity::Error); + return false; + } + report(APIEvent::Type::VSABufferFormatError, APIEvent::Severity::Error); + return std::nullopt; +} + +bool Device::findFirstExtendedVSAFromConsecutive(std::shared_ptr& record, uint64_t& pos, VSAParser& parser, std::optional optMetadata) { + static constexpr auto MaxReadAttempts = 10; + + VSAMetadata metadata; + if(!optMetadata) { + const auto& diskSize = getVSADiskSize(); + if(!diskSize) { + return false; + } + metadata.diskSize = *diskSize; + } else { + metadata = *optMetadata; + } + + // Reverse iteratively search from given pos for first record in sequence + const auto& index = record->getIndex(); + const auto& seqNum = record->getSequenceNum(); + pos -= (index - 1) * VSA::StandardRecordSize; + std::vector buffer; + buffer.resize(Disk::SectorSize); + uint16_t readCount = 0; + while(readCount < MaxReadAttempts) { + const auto& bytesRead = vsaReadLogicalDisk(pos, buffer.data(), Disk::SectorSize, metadata); + if(!bytesRead || *bytesRead < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + std::shared_ptr possibleFirstRecord; + const auto& possibleFirstStatus = parser.getRecordFromBytes(buffer.data(), Disk::SectorSize, possibleFirstRecord); + std::shared_ptr extendedVSA; + if(possibleFirstStatus == VSAParser::RecordParseStatus::NotARecordStart) { + // Most likely either null bytes or the middle of an AA6A record + pos -= VSA::StandardRecordSize; + readCount++; + continue; + } + extendedVSA = std::dynamic_pointer_cast(possibleFirstRecord); + if(!extendedVSA) { + // Record is not an extended message record + pos -= VSA::StandardRecordSize; + readCount++; + continue; + } + if(possibleFirstStatus == VSAParser::RecordParseStatus::Success && extendedVSA->getSequenceNum() == seqNum) { + // Found the desired record + record = extendedVSA; + return true; + } else if(possibleFirstStatus == VSAParser::RecordParseStatus::ConsecutiveExtended) { + if(seqNum != extendedVSA->getSequenceNum()) { // Another extended record sequence is intermixed with the one we want + pos -= VSA::StandardRecordSize; + } else { // Traverse backwards the minimum amount the first record must be behind extendedVSA + pos -= extendedVSA->getIndex() * VSA::StandardRecordSize; + } + } + readCount++; + } + report(APIEvent::Type::VSAMaxReadAttemptsReached, APIEvent::Severity::Error); + return false; +} + +bool Device::findPreviousRecordWithTimestamp(std::shared_ptr& record, uint64_t& pos, VSAParser& parser) { + static constexpr uint16_t MaxReadAttempts = 100; + static constexpr uint64_t ReadSize = 0x1000; + + std::vector buffer; + buffer.resize(ReadSize); + uint16_t readCount = 0; + while(readCount < MaxReadAttempts) { + pos -= ReadSize; + const auto& bytesRead = vsaReadLogicalDisk(pos, buffer.data(), ReadSize); + if(!bytesRead || *bytesRead < ReadSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + // Reverse search through buffer for a record with a valid timestamp + for (size_t offsetBufferEnd = VSA::StandardRecordSize; offsetBufferEnd < ReadSize; offsetBufferEnd += VSA::StandardRecordSize) { + const auto& parseStatus = parser.getRecordFromBytes(buffer.data() + ReadSize - offsetBufferEnd, offsetBufferEnd, record); + if(parseStatus == VSAParser::RecordParseStatus::Success) { + pos -= offsetBufferEnd; + return true; + } + } + readCount++; + } + report(APIEvent::Type::VSAMaxReadAttemptsReached, APIEvent::Severity::Error); + return false; // Exit if record not found within readCount number of reads +} + +bool Device::findVSAOffsetFromTimepoint(ICSClock::time_point point, uint64_t& vsaOffset, std::shared_ptr& record, + const VSAExtractionSettings& extractionSettings, std::optional optMetadata) { + static constexpr uint64_t LinearSearchTickDifference = 100000ull; // The maximum number of ticks offset from point at which to do a linear search + + // Grab important metadata features if metadata not defined + VSAMetadata metadata; + if(!optMetadata) { + if(!probeVSA(metadata, extractionSettings)) { + return false; + } + } else { + metadata = *optMetadata; + } + if(metadata.diskSize == 0) { // Disk size is unknown + report(APIEvent::Type::RequiredParameterNull, APIEvent::Severity::Error); + return false; + } + + uint64_t firstOffset = metadata.firstRecordLocation; // Offset of the first record chronologically + const auto& desiredTimestamp = VSA::getICSTimestampFromTimepoint(point); + VSAParser parser(report); + std::vector buffer; + buffer.resize(Disk::SectorSize); + + if(desiredTimestamp <= metadata.firstRecord->getTimestamp()) { + // Timestamp is less than first timestamp so we just return the first + vsaOffset = metadata.firstRecordLocation; + record = metadata.firstRecord; + return true; + } + + if(desiredTimestamp > metadata.lastRecord->getTimestamp()) { + report(APIEvent::Type::ParameterOutOfRange, APIEvent::Severity::Error); + return false; + } + + // Handle situation where first offset is not on sector boundary + // Find smallest tick difference between record in sector and desired timestamp + uint64_t bestTickDiff = UINT64_MAX; + std::shared_ptr bestRecord = nullptr; + uint64_t bestOffset = UINT64_MAX; + + auto findBestTickDiff = [&] (uint64_t readPos, uint64_t readSize) { + for(uint64_t i = 0; i < Disk::SectorSize; i += VSA::StandardRecordSize) { + std::shared_ptr testRecord; + const auto& testRecordStatus = parser.getRecordFromBytes(buffer.data() + i, static_cast(readSize - i), testRecord); + if(testRecordStatus == VSAParser::RecordParseStatus::Success) { + const auto& testTimestamp = testRecord->getTimestamp(); + uint64_t tickDiff = (testTimestamp > desiredTimestamp) + ? testTimestamp - desiredTimestamp + : desiredTimestamp - testTimestamp; + if(tickDiff < bestTickDiff) { + bestTickDiff = tickDiff; + bestOffset = readPos + i; + bestRecord = testRecord; + } + } + } + }; + + if(firstOffset % Disk::SectorSize != 0) { + // First record is not located at the beginning of a sector + // Find the best tick diff for the sector that the first record is in + // and ignore it during the binary search to allow for binary searching sectors, not bytes. + // Compare the bestTickDiff from the firstSector to results during the linear search to + // find the true best tick diff. + uint64_t readPos = firstOffset - (firstOffset % Disk::SectorSize); + const auto& bytesRead = vsaReadLogicalDisk(readPos, buffer.data(), Disk::SectorSize, metadata); + if(!bytesRead || *bytesRead < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + + findBestTickDiff(readPos, Disk::SectorSize); + + // Change the first offset to eliminate the first sector from the binary search + firstOffset = firstOffset + Disk::SectorSize - (firstOffset % Disk::SectorSize); + } + + uint64_t leftIndex = 0; // Index of the leftmost sector (not the byte) + uint64_t rightIndex = ((metadata.isOverlapped) ? metadata.diskSize : metadata.bufferEnd) - VSA::RecordStartOffset; + rightIndex = (rightIndex / Disk::SectorSize); + uint64_t midIndex; // Index of the middle sector (not the byte) + // Indices indicate sector offset from first sector location in metadata + while(leftIndex < rightIndex) { // Perform binary search for records with timestamp closest to the desired timestamp + midIndex = (rightIndex + leftIndex) / 2; + uint64_t readPos = firstOffset + midIndex * Disk::SectorSize; + const auto& bytesRead = vsaReadLogicalDisk(readPos, buffer.data(), Disk::SectorSize, metadata); + if(!bytesRead || *bytesRead < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + std::shared_ptr midRecord; + auto midRecordStatus = parser.getRecordFromBytes(buffer.data(), Disk::SectorSize, midRecord); + switch(midRecordStatus) { + case VSAParser::RecordParseStatus::NotARecordStart: + // This part of the buffer does not contain records + rightIndex = midIndex - 1; + continue; + case VSAParser::RecordParseStatus::ConsecutiveExtended: + // We dropped in the middle of an extended message record + auto extendedRecord = std::dynamic_pointer_cast(midRecord); + uint64_t pos = readPos; + if(!findFirstExtendedVSAFromConsecutive(extendedRecord, pos, parser, metadata)) { + pos = firstOffset + midIndex * Disk::SectorSize; + if(!findPreviousRecordWithTimestamp(midRecord, pos, parser)) { + return false; + } + midIndex = (pos - firstOffset) / Disk::SectorSize; + break; + } + midIndex = (pos - firstOffset) / Disk::SectorSize; + midRecord = extendedRecord; + break; + } + if(midIndex <= leftIndex) { + // Extended records cause problems with binary search + // Just move on to linear search + leftIndex = midIndex; + break; + } + if(midIndex > rightIndex) { + // Extended records cause problems with binary search + // Just move on to linear search + rightIndex = midIndex; + break; + } + if(!midRecord || !midRecord->isTimestampValid()) { // Unhandled failure to get timestamp + report(APIEvent::Type::VSATimestampNotFound, APIEvent::Severity::Error); + return false; + } + if(midRecord->getTimestamp() == desiredTimestamp) { + vsaOffset = firstOffset + midIndex * Disk::SectorSize; + record = midRecord; + return true; + } + if(midRecord->getTimestamp() < desiredTimestamp && midRecord->getTimestamp() > desiredTimestamp - LinearSearchTickDifference) { + // The timestamp of this sector is within desired linear search range + const uint64_t LinearReadByteAmount = 0x1000ull; // Number of bytes to read for linear search + uint64_t pos = firstOffset + midIndex * Disk::SectorSize; + const auto& recordBufferSize = metadata.diskSize - VSA::RecordStartOffset; + pos -= ((pos - VSA::RecordStartOffset) / recordBufferSize) * recordBufferSize; // Move pos to within physical space of record buffer + uint64_t lastTickDiff = UINT64_MAX; + buffer.resize(LinearReadByteAmount); + + // Begin linear search for record with closest timestamp to desired + while(lastTickDiff < LinearSearchTickDifference || lastTickDiff == UINT64_MAX) { + const auto& bytesReadLinear = vsaReadLogicalDisk(pos, buffer.data(), LinearReadByteAmount, metadata); + if(!bytesReadLinear || *bytesReadLinear < LinearReadByteAmount) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + for(uint64_t i = 0; i < LinearReadByteAmount; i += VSA::StandardRecordSize) { + std::shared_ptr testRecord; + const auto& testRecordStatus = parser.getRecordFromBytes(buffer.data() + i, static_cast(LinearReadByteAmount - i), testRecord); + if(testRecordStatus == VSAParser::RecordParseStatus::Success) { + const auto& testTimestamp = testRecord->getTimestamp(); + uint64_t tickDiff = (testTimestamp > desiredTimestamp) + ? testTimestamp - desiredTimestamp + : desiredTimestamp - testTimestamp; + if(tickDiff < bestTickDiff) { + bestTickDiff = tickDiff; + bestRecord = testRecord; + bestOffset = pos + i; + } + lastTickDiff = tickDiff; + if(testRecord->getType() == VSA::Type::AA6A) { + // Increment to skip the whole record if it is a full sector length record + i += Disk::SectorSize - VSA::StandardRecordSize; + } + } else if(testRecordStatus == VSAParser::RecordParseStatus::NotARecordStart) { + // We have reached an area with no records + break; + } else { + lastTickDiff = UINT64_MAX; + } + } + pos += LinearReadByteAmount; + if(!metadata.isOverlapped && pos > metadata.bufferEnd) { + // Ran out of valid records to search + break; + } + if(metadata.isOverlapped && pos >= metadata.firstRecordLocation && + pos - LinearReadByteAmount < metadata.firstRecordLocation) { + // Ran out of valid records to search + break; + } + } + // Return the location with the closest timestamp to the desired point + vsaOffset = bestOffset; + record = bestRecord; + return true; + } else if(midRecord->getTimestamp() < desiredTimestamp) { + // Keep right partition + leftIndex = midIndex; + } else { + // Keep left partition + rightIndex = midIndex; + } + } + + // Nothing found within desired linear search tick range + // Check for closest record to desired timestamp + if(leftIndex <= rightIndex) { + buffer.clear(); + uint64_t pos = firstOffset + leftIndex * Disk::SectorSize; + uint64_t bufferSize = (rightIndex - leftIndex + 1) * Disk::SectorSize; + buffer.resize(static_cast(bufferSize)); + const auto& bytesRead = vsaReadLogicalDisk(pos, buffer.data(), Disk::SectorSize, metadata); + if(!bytesRead || *bytesRead < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + findBestTickDiff(pos, bufferSize); + vsaOffset = bestOffset; + record = bestRecord; + return true; + } + report(APIEvent::Type::VSAOtherError, APIEvent::Severity::Error); + return false; +} + +std::optional Device::getVSATimestampOrBefore(VSAParser& parser, std::vector& buffer, uint64_t pos, uint64_t minPos, + std::optional optMetadata) { + VSAMetadata metadata; + if(!optMetadata) { + const auto& diskSize = getVSADiskSize(); + if(!diskSize) { + return std::nullopt; + } + metadata.diskSize = *diskSize; + } else { + metadata = *optMetadata; + } + + while(pos >= minPos) { + const auto& bytesRead = readLogicalDisk(pos, buffer.data(), Disk::SectorSize); + if(!bytesRead || *bytesRead < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return std::nullopt; + } + std::shared_ptr record; + auto parseStatus = parser.getRecordFromBytes(buffer.data(), Disk::SectorSize, record); + // Handle situations where we were dropped in the middle of an extended record + if(parseStatus == VSAParser::RecordParseStatus::ConsecutiveExtended) { + auto extendedRecord = std::dynamic_pointer_cast(record); + auto tempPos = pos; // If the first-extended search fails, we still have the original value of the search start location + if(!findFirstExtendedVSAFromConsecutive(extendedRecord, tempPos, parser, metadata)) { + // Unoptimized/Unloopable search + if(!findPreviousRecordWithTimestamp(record, pos, parser)) { + return std::nullopt; + } + return record->getTimestamp(); + } + return extendedRecord->getTimestamp(); + } + // Handle other situations without valid timestamp + if(parseStatus != VSAParser::RecordParseStatus::Success) { + pos -= VSA::StandardRecordSize; + } else { + return record->getTimestamp(); + } + } + report(APIEvent::Type::VSATimestampNotFound, APIEvent::Severity::Error); + return std::nullopt; +} + +std::optional Device::getVSATimestampOrAfter(VSAParser& parser, std::vector& buffer, uint64_t pos, uint64_t maxPos, + std::optional optMetadata) { + VSAMetadata metadata; + if(!optMetadata) { + const auto& diskSize = getVSADiskSize(); + if(!diskSize) { + return std::nullopt; + } + metadata.diskSize = *diskSize; + } else { + metadata = *optMetadata; + } + + while(pos <= maxPos) { + const auto& bytesRead = readLogicalDisk(pos, buffer.data(), Disk::SectorSize); + if(!bytesRead || *bytesRead < Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return std::nullopt; + } + std::shared_ptr record; + auto parseStatus = parser.getRecordFromBytes(buffer.data(), Disk::SectorSize, record); + if(parseStatus == VSAParser::RecordParseStatus::ConsecutiveExtended) { + auto extendedRecord = std::dynamic_pointer_cast(record); + auto tempPos = pos; // If the first-extended search fails, we still have the original value of the search start location + if(!findFirstExtendedVSAFromConsecutive(extendedRecord, tempPos, parser, metadata)) { + pos += VSA::StandardRecordSize; + continue; + } + return extendedRecord->getTimestamp(); + } + // Other situations with invalid timestamp + if(parseStatus != VSAParser::RecordParseStatus::Success) { + pos += VSA::StandardRecordSize; + } else { + return record->getTimestamp(); + } + } + report(APIEvent::Type::VSATimestampNotFound, APIEvent::Severity::Error); + return std::nullopt; +} + +bool Device::parseVSA( + VSAMetadata& metadata, const VSAExtractionSettings& extractionSettings, const VSAMessageReadFilter& filter +) { + static constexpr uint64_t MaxReadAmountPerIteration = 0x10000; // Read 512 KB per iteration + static constexpr uint64_t MaxReadFailuresAllowed = 10; + + if(filter.readRange.first > filter.readRange.second) { + // First timestamp occurs after second timestamp in filter + // Do not fail to allow for other filters to do work + report(APIEvent::Type::ParameterOutOfRange, APIEvent::Severity::EventWarning); + return true; + } + + // Location of first time_point from the filter + uint64_t readOffset; + std::shared_ptr recordAtOffset; + if(!findVSAOffsetFromTimepoint(filter.readRange.first, readOffset, recordAtOffset, extractionSettings, metadata)) { + report(APIEvent::Type::VSATimestampNotFound, APIEvent::Severity::Error); + return false; + } + if(readOffset >= metadata.diskSize) { + readOffset -= metadata.diskSize - VSA::RecordStartOffset; + } + + std::vector buffer; + bool success = true; + bool moreToRead = true; + uint64_t numBytesRead = 0; // The number of bytes that have been read from the vsa buffer for this parse call + VSAParser::Settings parserSettings = VSAParser::Settings::messageRecords(); + VSAParser parser(report, parserSettings); + parser.setMessageFilter(filter); + while(moreToRead) { + uint64_t amount; + if(!metadata.isOverlapped) { + moreToRead = readOffset + MaxReadAmountPerIteration < metadata.bufferEnd; + amount = std::min(MaxReadAmountPerIteration, metadata.bufferEnd - readOffset); + } else { + uint64_t readEnd = readOffset + MaxReadAmountPerIteration; + if(readEnd > metadata.diskSize) { + // Make sure read end is within the buffer for a possible looped read + readEnd -= metadata.diskSize - VSA::RecordStartOffset; + } + moreToRead = !(readOffset < metadata.bufferEnd && readEnd >= metadata.bufferEnd); + amount = moreToRead ? MaxReadAmountPerIteration : metadata.bufferEnd - readOffset; + } + if(amount < VSA::StandardRecordSize) { + break; + } + + buffer.resize(static_cast(amount)); + uint64_t readAttempt = 1; + while(true) { + const auto& bytesRead = vsaReadLogicalDisk(readOffset, buffer.data(), amount, metadata); + if(bytesRead && *bytesRead == amount) { + break; + } + if(readAttempt >= MaxReadFailuresAllowed) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return false; + } + report(APIEvent::Type::FailedToRead, APIEvent::Severity::EventWarning); + ++readAttempt; + } + + numBytesRead += amount; + success = parser.parseBytes(buffer.data(), amount); + if(!success) { + report(APIEvent::Type::VSAByteParseFailure, APIEvent::Severity::Error); + return false; + } + if(parser.empty()) { // No message records or all extended message records were not terminated in block + readOffset += amount; + continue; + } + const auto& lastRecord = parser.back(); + if(lastRecord->getTimestampICSClock() >= filter.readRange.second) { + // Reached the desired end timestamp + moreToRead = false; + } + if(!dispatchVSAMessages(parser)) { // Piecemeal dispatch messages so we don't have exceptionally large vectors + return false; + } + readOffset += amount; + } + return success; +} + +std::optional Device::vsaReadLogicalDisk(uint64_t pos, uint8_t* into, uint64_t amount, std::optional metadata) { + uint64_t diskSize; + if(metadata) { + diskSize = metadata->diskSize; + } else { + const auto& testDiskSize = getVSADiskSize(); + if(!testDiskSize) { + return std::nullopt; + } + diskSize = *testDiskSize; + } + // Set the position to be within the ring buffer + if(pos < VSA::RecordStartOffset) { + pos = diskSize - (VSA::RecordStartOffset - pos); + } else if(pos >= diskSize) { + const auto& bufferSize = diskSize - VSA::RecordStartOffset; + const auto& timesLooped = (pos - VSA::RecordStartOffset) / bufferSize; + pos -= bufferSize * timesLooped; + } + + if(amount > diskSize - VSA::RecordStartOffset) { // Given read amount is too large + amount = diskSize - VSA::RecordStartOffset; // Do full disk dump + } + if(pos + amount < diskSize) { // Read doesn't need to loop + return readLogicalDisk(pos, into, amount); + } + uint64_t firstReadAmount = diskSize - pos; + if(!readLogicalDisk(pos, into, firstReadAmount)) { + return std::nullopt; + } + return readLogicalDisk(VSA::RecordStartOffset, into + firstReadAmount, amount - firstReadAmount); +} + +bool Device::dispatchVSAMessages(VSAParser& parser) { + std::vector> packets; + if(!parser.extractMessagePackets(packets)) { + report(APIEvent::Type::VSAByteParseFailure, APIEvent::Severity::Error); + return false; + } + for(const auto& packet : packets) { + std::shared_ptr msg; + if(!com->decoder->decode(msg, packet)) { + return false; + } + com->dispatchMessage(msg); + } + return true; +} + +std::optional Device::getCoreMiniScriptTimestamp() { + static constexpr auto CoreMiniTimestampLocation = 48; + static constexpr auto CoreMiniTimestampSize = 8; + uint8_t buffer[CoreMiniTimestampSize]; + const auto& numBytes = readLogicalDisk(CoreMiniTimestampLocation, buffer, CoreMiniTimestampSize); + if(!numBytes || *numBytes < CoreMiniTimestampSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return std::nullopt; + } + return *reinterpret_cast(buffer); +} + +std::optional Device::getVSADiskSize() { + uint64_t diskSize; + auto scriptStatus = getScriptStatus(); + if(!scriptStatus) { + return std::nullopt; + } + if(!scriptStatus->isCoreminiRunning) { + startScript(); + scriptStatus = getScriptStatus(); + if(!scriptStatus) { + return std::nullopt; + } + diskSize = (scriptStatus->maxSector + 1) * Disk::SectorSize; + stopScript(); + } else { + diskSize = (scriptStatus->maxSector + 1) * Disk::SectorSize; + } + if(diskSize == Disk::SectorSize) { + report(APIEvent::Type::FailedToRead, APIEvent::Severity::Error); + return std::nullopt; + } + return diskSize; } \ No newline at end of file diff --git a/disk/vsa/vsa.cpp b/disk/vsa/vsa.cpp new file mode 100644 index 0000000..6684cb5 --- /dev/null +++ b/disk/vsa/vsa.cpp @@ -0,0 +1,39 @@ +#include "icsneo/disk/vsa/vsa.h" +#include "icsneo/communication/packet/ethernetpacket.h" + +using namespace icsneo; + +// VSA Base Class Functions + +// VSAMessage Class Functions +std::shared_ptr VSAMessage::getPacket() const +{ + auto packet = std::make_shared(); + packet->network = network; + reservePacketData(packet); + packet->data.insert(packet->data.end(), payload.begin(), payload.end()); + return packet; +} + +// VSAExtendedMessage Class Functions + +void VSAExtendedMessage::appendPacket(std::shared_ptr packet) const +{ + packet->data.insert(packet->data.end(), payload.begin(), payload.end()); + // Set the network if not already set (Happens in AA0F records) + if(packet->network.getNetID() == Network::NetID::Invalid) { + packet->network = network; + } +} + +void VSAExtendedMessage::truncatePacket(std::shared_ptr packet) +{ + static constexpr auto EthernetLengthOffset = 26u; + switch(packet->network.getType()) { + case Network::Type::Ethernet: + const auto& packetLength = *reinterpret_cast(packet->data.data() + EthernetLengthOffset); + const size_t ethernetFrameSize = packetLength - (sizeof(uint16_t) * 2); + const size_t bytestreamExpectedSize = sizeof(HardwareEthernetPacket) + ethernetFrameSize; + packet->data.resize(bytestreamExpectedSize); + } +} \ No newline at end of file diff --git a/disk/vsa/vsa02.cpp b/disk/vsa/vsa02.cpp new file mode 100644 index 0000000..a0fc997 --- /dev/null +++ b/disk/vsa/vsa02.cpp @@ -0,0 +1,26 @@ +#include "icsneo/disk/vsa/vsa02.h" + +using namespace icsneo; + +VSA02::VSA02(uint8_t* const recordBytes) + : VSA() +{ + setType(VSA::Type::AA02); + constantIndex = *reinterpret_cast(recordBytes + 2); + flags = *reinterpret_cast(recordBytes + 4); + pieceCount = recordBytes[5]; + timestamp = *reinterpret_cast(recordBytes + 6) & UINT63_MAX; + samples.insert(samples.end(), recordBytes + 14, recordBytes + 30); + checksum = *reinterpret_cast(recordBytes + 30); + doChecksum(recordBytes); +} + +void VSA02::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 15; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} \ No newline at end of file diff --git a/disk/vsa/vsa03.cpp b/disk/vsa/vsa03.cpp new file mode 100644 index 0000000..e7a8f6b --- /dev/null +++ b/disk/vsa/vsa03.cpp @@ -0,0 +1,24 @@ +#include "icsneo/disk/vsa/vsa03.h" + +using namespace icsneo; + +VSA03::VSA03(uint8_t* const recordBytes) + : VSA() +{ + setType(VSA::Type::AA03); + eventType = static_cast(*reinterpret_cast(recordBytes + 2)); + eventData = *reinterpret_cast(recordBytes + 4); + timestamp = *reinterpret_cast(recordBytes + 6) & UINT63_MAX; + checksum = *reinterpret_cast(recordBytes + 14); + doChecksum(recordBytes); +} + +void VSA03::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 7; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} \ No newline at end of file diff --git a/disk/vsa/vsa04.cpp b/disk/vsa/vsa04.cpp new file mode 100644 index 0000000..8e14a39 --- /dev/null +++ b/disk/vsa/vsa04.cpp @@ -0,0 +1,24 @@ +#include "icsneo/disk/vsa/vsa04.h" + +using namespace icsneo; + +VSA04::VSA04(uint8_t* const recordBytes) + : VSA() +{ + setType(VSA::Type::AA04); + flags = *reinterpret_cast(recordBytes + 2); + partitionIndex = *reinterpret_cast(recordBytes + 4); + timestamp = *reinterpret_cast(recordBytes + 6) & UINT63_MAX; + checksum = *reinterpret_cast(recordBytes + 14); + doChecksum(recordBytes); +} + +void VSA04::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 7; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} \ No newline at end of file diff --git a/disk/vsa/vsa05.cpp b/disk/vsa/vsa05.cpp new file mode 100644 index 0000000..32b61df --- /dev/null +++ b/disk/vsa/vsa05.cpp @@ -0,0 +1,23 @@ +#include "icsneo/disk/vsa/vsa05.h" + +using namespace icsneo; + +VSA05::VSA05(uint8_t* const recordBytes) + : VSA() +{ + setType(VSA::Type::AA05); + errorType = static_cast(*reinterpret_cast(recordBytes + 2)); + errorNetwork = *reinterpret_cast(recordBytes + 4); + timestamp = *reinterpret_cast(recordBytes + 6) & UINT63_MAX; + checksum = *reinterpret_cast(recordBytes + 14); +} + +void VSA05::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 7; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} \ No newline at end of file diff --git a/disk/vsa/vsa06.cpp b/disk/vsa/vsa06.cpp new file mode 100644 index 0000000..7b9194c --- /dev/null +++ b/disk/vsa/vsa06.cpp @@ -0,0 +1,25 @@ +#include "icsneo/disk/vsa/vsa06.h" + +using namespace icsneo; + +VSA06::VSA06(uint8_t* const recordBytes) + : VSA() +{ + setType(VSA::Type::AA06); + savedSectors.insert(savedSectors.end(), reinterpret_cast(recordBytes + 2), reinterpret_cast(recordBytes + 18)); + error = *reinterpret_cast(recordBytes + 18); + savedSectorsHigh = *reinterpret_cast(recordBytes + 20); + timestamp = *reinterpret_cast(recordBytes + 22) & UINT63_MAX; + checksum = *reinterpret_cast(recordBytes + 30); + doChecksum(recordBytes); +} + +void VSA06::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 15; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} \ No newline at end of file diff --git a/disk/vsa/vsa07.cpp b/disk/vsa/vsa07.cpp new file mode 100644 index 0000000..71673ba --- /dev/null +++ b/disk/vsa/vsa07.cpp @@ -0,0 +1,25 @@ +#include "icsneo/disk/vsa/vsa07.h" + +using namespace icsneo; + +VSA07::VSA07(uint8_t* const recordBytes) + : VSA() +{ + setType(VSA::Type::AA07); + lastSector = *reinterpret_cast(recordBytes + 2); + currentSector = *reinterpret_cast(recordBytes + 6); + reserved.insert(reserved.end(), recordBytes + 10, recordBytes + 22); + timestamp = *reinterpret_cast(recordBytes + 22) & UINT63_MAX; + checksum = *reinterpret_cast(recordBytes + 30); + doChecksum(recordBytes); +} + +void VSA07::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 15; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} \ No newline at end of file diff --git a/disk/vsa/vsa08.cpp b/disk/vsa/vsa08.cpp new file mode 100644 index 0000000..acfedd9 --- /dev/null +++ b/disk/vsa/vsa08.cpp @@ -0,0 +1,24 @@ +#include "icsneo/disk/vsa/vsa08.h" + +using namespace icsneo; + +VSA08::VSA08(uint8_t* const recordBytes) + : VSA() +{ + setType(VSA::Type::AA08); + troubleSramCount.insert(troubleSramCount.end(), recordBytes + 2, recordBytes + 6); + troubleSectors.insert(troubleSectors.end(), reinterpret_cast(recordBytes + 6), reinterpret_cast(recordBytes + 20)); + timestamp = *reinterpret_cast(recordBytes + 22) & UINT63_MAX; + checksum = *reinterpret_cast(recordBytes + 30); + doChecksum(recordBytes); +} + +void VSA08::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 15; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} \ No newline at end of file diff --git a/disk/vsa/vsa09.cpp b/disk/vsa/vsa09.cpp new file mode 100644 index 0000000..9425c1e --- /dev/null +++ b/disk/vsa/vsa09.cpp @@ -0,0 +1,32 @@ +#include "icsneo/disk/vsa/vsa09.h" + +using namespace icsneo; + +VSA09::VSA09(uint8_t* const recordBytes) + : VSA() +{ + setType(VSA::Type::AA09); + serialNumber = *reinterpret_cast(recordBytes + 2); + firmwareMajorVersion = recordBytes[6]; + firmwareMinorVersion = recordBytes[7]; + manufactureMajorRevision = recordBytes[8]; + manufactureMinorRevision = recordBytes[9]; + bootloaderMajorVersion = recordBytes[10]; + bootloaderMinorVersion = recordBytes[11]; + reserved0.insert(reserved0.end(), recordBytes + 12, recordBytes + 18); + hardwareID = static_cast(recordBytes[18]); + reserved1.insert(reserved1.end(), recordBytes + 19, recordBytes + 22); + timestamp = *reinterpret_cast(recordBytes + 22) & UINT63_MAX; + checksum = *reinterpret_cast(recordBytes + 30); + doChecksum(recordBytes); +} + +void VSA09::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 15; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} \ No newline at end of file diff --git a/disk/vsa/vsa0b.cpp b/disk/vsa/vsa0b.cpp new file mode 100644 index 0000000..fdb2537 --- /dev/null +++ b/disk/vsa/vsa0b.cpp @@ -0,0 +1,38 @@ +#include "icsneo/disk/vsa/vsa0b.h" + +#include + +using namespace icsneo; + +static constexpr auto PayloadOffset = 4; + +VSA0B::VSA0B(uint8_t* const recordBytes) + : VSAMessage(recordBytes + PayloadOffset, CoreMiniPayloadSize, static_cast(recordBytes[29])) +{ + setType(VSA::Type::AA0B); + captureBitfield = reinterpret_cast(recordBytes)[1]; + timestamp = *reinterpret_cast(recordBytes + 20) & UINT63_MAX; + reserved = recordBytes[28]; + checksum = reinterpret_cast(recordBytes)[15]; + doChecksum(recordBytes); +} + +void VSA0B::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 15; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} + +bool VSA0B::filter(const std::shared_ptr filter) +{ + if((filter->captureBitfield != captureBitfield && filter->captureBitfield != UINT16_MAX) || + getICSTimestampFromTimepoint(filter->readRange.first) > timestamp || + getICSTimestampFromTimepoint(filter->readRange.second) < timestamp) { + return false; + } + return true; +} \ No newline at end of file diff --git a/disk/vsa/vsa0c.cpp b/disk/vsa/vsa0c.cpp new file mode 100644 index 0000000..1f3c33e --- /dev/null +++ b/disk/vsa/vsa0c.cpp @@ -0,0 +1,27 @@ +#include "icsneo/disk/vsa/vsa0c.h" + +using namespace icsneo; + +VSA0C::VSA0C(uint8_t* const recordBytes) + : VSA() +{ + setType(VSA::Type::AA0C); + captureBitfield = *reinterpret_cast(recordBytes + 2); + audioPreamble = recordBytes[4]; + audioHeader = recordBytes[5]; + pcmData.insert(pcmData.end(), recordBytes + 6, recordBytes + 20); + timestamp = *reinterpret_cast(recordBytes + 20) & UINT63_MAX; + vNetBitfield = *reinterpret_cast(recordBytes + 28); + checksum = *reinterpret_cast(recordBytes + 30); + doChecksum(recordBytes); +} + +void VSA0C::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 15; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} \ No newline at end of file diff --git a/disk/vsa/vsa0d.cpp b/disk/vsa/vsa0d.cpp new file mode 100644 index 0000000..76beda2 --- /dev/null +++ b/disk/vsa/vsa0d.cpp @@ -0,0 +1,104 @@ +#include "icsneo/disk/vsa/vsa0d.h" + +#include + +using namespace icsneo; + +static constexpr auto FirstPayloadOffset = 8; +static constexpr auto FirstPayloadSize = 12; +static constexpr auto ConsecutivePayloadOffset = 4; +static constexpr auto LastPayloadSize = 24; +static constexpr auto OtherPayloadSize = 28; + +// Parent class functions + +VSA0D::VSA0D(uint8_t* const recordBytes, uint8_t* const messageBytes, size_t numBytes, uint32_t& runningChecksum, Network::CoreMini networkId) + : VSAExtendedMessage(messageBytes, numBytes, networkId) +{ + static constexpr auto DWordSize = 4; + setType(VSA::Type::AA0D); + setIndex(*reinterpret_cast(recordBytes + 2) & 0x01FFu); + setSequenceNum((*reinterpret_cast(recordBytes + 2) & 0xFE00u) >> 9); + uint32_t* dwords = reinterpret_cast(payload.data()); + for(size_t i = 0; i < payload.size() / DWordSize; i++) { + runningChecksum += dwords[i]; + } +} + +// First Record Functions + +VSA0DFirst::VSA0DFirst(uint8_t* const recordBytes, uint32_t& runningChecksum) + : VSA0D(recordBytes, recordBytes + FirstPayloadOffset, FirstPayloadSize, runningChecksum, static_cast(recordBytes[29])) +{ + captureBitfield = *reinterpret_cast(recordBytes + 4); + setRecordCount(*reinterpret_cast(recordBytes + 6)); + timestamp = *reinterpret_cast(recordBytes + 20) & UINT63_MAX; + vNetInfo = *reinterpret_cast(recordBytes + 28); + checksum = *reinterpret_cast(recordBytes + 30); + doChecksum(recordBytes); + uint32_t* const timestampDWords = reinterpret_cast(timestamp); + runningChecksum += timestampDWords[0]; + runningChecksum += timestampDWords[1]; +} + +void VSA0DFirst::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 15; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} + +void VSA0DFirst::reservePacketData(std::shared_ptr& packet) const +{ + uint32_t numMessageBytes = (getRecordCount() - 2) * OtherPayloadSize + FirstPayloadSize + LastPayloadSize; + packet->data.reserve(numMessageBytes); +} + +void VSA0DFirst::reorderPayload(std::vector& secondPayload) +{ + std::vector tempPayload; + tempPayload.insert(tempPayload.end(), secondPayload.begin(), secondPayload.begin() + 4); + uint8_t* timestampBytes = reinterpret_cast(×tamp); + tempPayload.insert(tempPayload.end(), timestampBytes, timestampBytes + 8); + tempPayload.insert(tempPayload.end(), secondPayload.begin() + 4, secondPayload.end()); + payload.clear(); + secondPayload.clear(); + payload.insert(payload.end(), tempPayload.begin(), tempPayload.begin() + 12); // This is done because the capacity of payload is already 12 + secondPayload.insert(secondPayload.end(), tempPayload.begin() + 12, tempPayload.end()); +} + +bool VSA0DFirst::filter(const std::shared_ptr filter) +{ + if((filter->captureBitfield != captureBitfield && filter->captureBitfield != UINT16_MAX) || + getICSTimestampFromTimepoint(filter->readRange.first) > timestamp || + getICSTimestampFromTimepoint(filter->readRange.second) < timestamp) { + return false; + } + return true; +} + +// Consecutive Record Functions + +VSA0DConsecutive::VSA0DConsecutive(uint8_t* const recordBytes, uint32_t& runningChecksum, std::shared_ptr first, bool isLastRecord) + : VSA0D(recordBytes, recordBytes + ConsecutivePayloadOffset, isLastRecord ? LastPayloadSize : OtherPayloadSize, runningChecksum) +{ + this->first = first; + calculatedChecksum = runningChecksum; + if(getIndex() == 1) { + first->reorderPayload(payload); + } else if(isLastRecord) { + recordChecksum = *reinterpret_cast(recordBytes + 28); + doChecksum(recordBytes); + } else { + setChecksumFailed(first->getChecksumFailed()); + } + setRecordCount(first->getRecordCount()); +} + +void VSA0DConsecutive::doChecksum(uint8_t* recordBytes) +{ + setChecksumFailed(recordBytes && calculatedChecksum != recordChecksum); +} \ No newline at end of file diff --git a/disk/vsa/vsa0e.cpp b/disk/vsa/vsa0e.cpp new file mode 100644 index 0000000..71d120f --- /dev/null +++ b/disk/vsa/vsa0e.cpp @@ -0,0 +1,111 @@ +#include "icsneo/disk/vsa/vsa0e.h" + +#include + +using namespace icsneo; + +static constexpr auto FirstPayloadOffset = 10; +static constexpr auto FirstPayloadSize = 10; +static constexpr auto ConsecutivePayloadOffset = 4; +static constexpr auto LastPayloadSize = 24; +static constexpr auto OtherPayloadSize = 28; + +// Parent class functions + +VSA0E::VSA0E(uint8_t* const recordBytes, uint8_t* const messageBytes, size_t numBytes, uint32_t& runningChecksum, Network::CoreMini networkId) + : VSAExtendedMessage(messageBytes, numBytes, networkId) +{ + static constexpr auto DWordSize = 4; + setType(VSA::Type::AA0E); + setIndex(static_cast(recordBytes[2])); + setSequenceNum(static_cast(recordBytes[3])); + if(getIndex() == 0) { + runningChecksum = (static_cast(payload[0]) << 16) | (static_cast(payload[1]) << 24); + uint32_t* dwords = reinterpret_cast(payload.data() + 2); + for(size_t i = 0; i < (payload.size() - 2) / DWordSize; i++) { + runningChecksum += dwords[i]; + } + } else { + uint32_t* dwords = reinterpret_cast(payload.data()); + for(size_t i = 0; i < payload.size() / DWordSize; i++) { + runningChecksum += dwords[i]; + } + } +} + +// First Record Functions + +VSA0EFirst::VSA0EFirst(uint8_t* const recordBytes, uint32_t& runningChecksum) + : VSA0E(recordBytes, recordBytes + FirstPayloadOffset, FirstPayloadSize, runningChecksum, + static_cast(*reinterpret_cast(recordBytes + 28))) +{ + captureBitfield = *reinterpret_cast(recordBytes + 4); + setRecordCount(*reinterpret_cast(recordBytes + 6)); + timestamp = *reinterpret_cast(recordBytes + 20) & UINT63_MAX; + checksum = *reinterpret_cast(recordBytes + 30); + doChecksum(recordBytes); +} + +void VSA0EFirst::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for(size_t i = 0; i < 15; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} + +void VSA0EFirst::reservePacketData(std::shared_ptr& packet) const +{ + uint32_t numMessageBytes = (getRecordCount() - 2) * OtherPayloadSize + FirstPayloadSize + LastPayloadSize; + packet->data.reserve(numMessageBytes); +} + +bool VSA0EFirst::filter(const std::shared_ptr filter) +{ + if((filter->captureBitfield != captureBitfield && filter->captureBitfield != UINT16_MAX) || + getICSTimestampFromTimepoint(filter->readRange.first) > timestamp || + getICSTimestampFromTimepoint(filter->readRange.second) < timestamp) { + return false; + } + return true; +} + +void VSA0EFirst::reorderPayload(std::vector& secondPayload) +{ + std::vector tempPayload; + tempPayload.insert(tempPayload.end(), { 0, 0 }); // Leaving this in here temporarily to figure out checksum stuff + tempPayload.insert(tempPayload.end(), payload.begin(), payload.end()); + tempPayload.insert(tempPayload.end(), secondPayload.begin(), secondPayload.begin() + 6); + uint8_t* timestampBytes = reinterpret_cast(×tamp); + tempPayload.insert(tempPayload.end(), timestampBytes, timestampBytes + 8); + tempPayload.insert(tempPayload.end(), secondPayload.begin() + 6, secondPayload.end()); + payload.clear(); + secondPayload.clear(); + payload.insert(payload.end(), tempPayload.begin(), tempPayload.begin() + 10); // This is done because the capacity of payload is already 10 + secondPayload.insert(secondPayload.end(), tempPayload.begin() + 12, tempPayload.end()); +} + +// Consecutive Record Functions + +VSA0EConsecutive::VSA0EConsecutive(uint8_t* const recordBytes, uint32_t& runningChecksum, std::shared_ptr first, bool isLastRecord) + : VSA0E(recordBytes, recordBytes + ConsecutivePayloadOffset, isLastRecord ? LastPayloadSize : OtherPayloadSize, runningChecksum) +{ + this->first = first; + calculatedChecksum = runningChecksum; + if(getIndex() == 1) { + first->reorderPayload(payload); + } else if(isLastRecord) { + recordChecksum = *reinterpret_cast(recordBytes + 28); + doChecksum(recordBytes); + } else { + setChecksumFailed(first->getChecksumFailed()); + } + setRecordCount(first->getRecordCount()); +} + +void VSA0EConsecutive::doChecksum(uint8_t* recordBytes) +{ + setChecksumFailed(recordBytes && calculatedChecksum != recordChecksum); +} \ No newline at end of file diff --git a/disk/vsa/vsa0f.cpp b/disk/vsa/vsa0f.cpp new file mode 100644 index 0000000..72254d6 --- /dev/null +++ b/disk/vsa/vsa0f.cpp @@ -0,0 +1,106 @@ +#include "icsneo/disk/vsa/vsa0f.h" + +#include + +using namespace icsneo; + +static constexpr auto FirstPayloadOffset = 18; +static constexpr auto FirstPayloadSize = 14; +static constexpr auto LastPayloadSize = 24; +static constexpr auto OtherPayloadSize = 28; + +// Parent class functions + +VSA0F::VSA0F(uint8_t* const recordBytes, uint8_t* const messageBytes, size_t numBytes, uint32_t& runningChecksum, Network::CoreMini networkId) + : VSAExtendedMessage(messageBytes, numBytes, networkId) +{ + static constexpr auto DWordSize = 4; + setType(VSA::Type::AA0F); + setIndex(*reinterpret_cast(recordBytes + 2) & 0x01FFu); + setSequenceNum((*reinterpret_cast(recordBytes + 2) & 0xFE00u) >> 9); + if(getIndex() == 0) { + runningChecksum = (static_cast(payload[0]) << 16) | (static_cast(payload[1]) << 24); + uint32_t* dwords = reinterpret_cast(payload.data() + 2); + for (size_t i = 0; i < (payload.size() - 2) / DWordSize; i++) { + runningChecksum += dwords[i]; + } + } else { + uint32_t* dwords = reinterpret_cast(recordBytes); + for (size_t i = 0; i < 8; i++) { + runningChecksum += dwords[i]; + } + } +} + +// First Record Functions + +VSA0FFirst::VSA0FFirst(uint8_t* const recordBytes, uint32_t& runningChecksum) + : VSA0F(recordBytes, recordBytes + FirstPayloadOffset, FirstPayloadSize, runningChecksum) +{ + captureBitfield = *reinterpret_cast(recordBytes + 4); + uint16_t byteCount = *reinterpret_cast(recordBytes + 6); + uint16_t recordCount; + if(byteCount <= FirstPayloadSize) { + recordCount = 1; + } else if(byteCount <= FirstPayloadSize + LastPayloadSize) { + recordCount = 2; + } else { + byteCount -= FirstPayloadSize + LastPayloadSize; + recordCount = 2 + byteCount / OtherPayloadSize; + if (byteCount % OtherPayloadSize != 0) { + recordCount += 1; + } + } + setRecordCount(recordCount); + timestamp = *reinterpret_cast(recordBytes + 8) & UINT63_MAX; + checksum = *reinterpret_cast(recordBytes + 16); + doChecksum(recordBytes); + // Network ID is not found in first record for AA0F + // Only the subsequent records have the Network ID in the payload +} + +void VSA0FFirst::doChecksum(uint8_t* recordBytes) +{ + uint16_t* words = reinterpret_cast(recordBytes); + uint16_t sum = 0; + for (size_t i = 0; i < 15; i++) { + sum += words[i]; + } + setChecksumFailed(sum != checksum); +} + +void VSA0FFirst::reservePacketData(std::shared_ptr& packet) const +{ + uint32_t numMessageBytes = (getRecordCount() - 2) * OtherPayloadSize + FirstPayloadSize + LastPayloadSize; + packet->data.reserve(numMessageBytes); +} + +bool VSA0FFirst::filter(const std::shared_ptr filter) +{ + if(filter->captureBitfield != captureBitfield || + getICSTimestampFromTimepoint(filter->readRange.first) > timestamp || + getICSTimestampFromTimepoint(filter->readRange.second) < timestamp) { + return false; + } + return true; +} + +// Consecutive Record Functions + +VSA0FConsecutive::VSA0FConsecutive(uint8_t* const recordBytes, uint32_t& runningChecksum, std::shared_ptr first, bool isLastRecord) + : VSA0F(recordBytes, recordBytes + 4, isLastRecord ? LastPayloadSize : OtherPayloadSize, runningChecksum) +{ + this->first = first; + calculatedChecksum = runningChecksum; + if(isLastRecord) { + doChecksum(recordBytes); + } else { + network = Network(static_cast(*reinterpret_cast(recordBytes + 28))); // Network ID is stored in 25th and 26th recordBytes of payload + } + setRecordCount(first->getRecordCount()); +} + +void VSA0FConsecutive::doChecksum(uint8_t* recordBytes) +{ + setChecksumFailed(recordBytes && calculatedChecksum != 0); +} \ No newline at end of file diff --git a/disk/vsa/vsa6a.cpp b/disk/vsa/vsa6a.cpp new file mode 100644 index 0000000..b91c956 --- /dev/null +++ b/disk/vsa/vsa6a.cpp @@ -0,0 +1,37 @@ +#include "icsneo/disk/vsa/vsa6a.h" +#include "icsneo/disk/diskdriver.h" + +using namespace icsneo; + +static constexpr auto PayloadOffset = 56; +static constexpr auto PayloadSize = 452; +static constexpr auto TimestampOffset = 48; +static constexpr auto TimestampSize = 8; + +VSA6A::VSA6A(uint8_t* const recordBytes) + : VSA() +{ + setType(VSA::Type::AA6A); + + sequenceNum = *reinterpret_cast(recordBytes + 34); + totalSectors = *reinterpret_cast(recordBytes + 38); + reserved = *reinterpret_cast(recordBytes + 42); + timestamp = *reinterpret_cast(recordBytes + 46) & UINT63_MAX; + timestampSum = *reinterpret_cast(recordBytes + 54); + data.insert(data.end(), recordBytes + 56, recordBytes + 508); + checksum = *reinterpret_cast(recordBytes + 508); + doChecksum(recordBytes); +} + +void VSA6A::doChecksum(uint8_t* recordBytes) +{ + uint32_t sum = 0; + for(size_t i = PayloadOffset; i < PayloadOffset+ PayloadSize; i++) { + sum += recordBytes[i]; + } + uint16_t tSum = 0; + for(size_t i = TimestampOffset; i < TimestampOffset + TimestampSize; i++) { + tSum += recordBytes[i]; + } + setChecksumFailed(sum != checksum || tSum != timestampSum); +} \ No newline at end of file diff --git a/disk/vsa/vsaparser.cpp b/disk/vsa/vsaparser.cpp new file mode 100644 index 0000000..008dff0 --- /dev/null +++ b/disk/vsa/vsaparser.cpp @@ -0,0 +1,460 @@ +#include "icsneo/disk/vsa/vsaparser.h" +#include "icsneo/disk/vsa/vsa02.h" +#include "icsneo/disk/vsa/vsa03.h" +#include "icsneo/disk/vsa/vsa04.h" +#include "icsneo/disk/vsa/vsa05.h" +#include "icsneo/disk/vsa/vsa06.h" +#include "icsneo/disk/vsa/vsa07.h" +#include "icsneo/disk/vsa/vsa08.h" +#include "icsneo/disk/vsa/vsa09.h" +#include "icsneo/disk/vsa/vsa0b.h" +#include "icsneo/disk/vsa/vsa0c.h" +#include "icsneo/disk/vsa/vsa0d.h" +#include "icsneo/disk/vsa/vsa0e.h" +#include "icsneo/disk/vsa/vsa0f.h" +#include "icsneo/disk/vsa/vsa6a.h" +#include "icsneo/disk/diskdriver.h" + +#include + +using namespace icsneo; + +bool VSAParser::parseBytes(uint8_t* const bytes, uint64_t arrLen) +{ + uint64_t bytesOffset = 0; + while(bytesOffset + VSA::StandardRecordSize <= arrLen) { // Enough bytes to read for Standard Record + if(bytes[bytesOffset] != 0xAAu) { + // Invalid Input + return false; + } + switch(bytes[bytesOffset + 1]) { + case 0x00u: // Pad Record + bytesOffset += VSA::StandardRecordSize; + break; + case 0x01u: // Message Data (Deprecated) + hasDeprecatedRecords = true; + bytesOffset += VSA::StandardRecordSize; + break; + case 0x02u: // Logdata Record + if(settings.extractAA02) { + vsaRecords.push_back(std::make_shared(bytes + bytesOffset)); + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x03u: // Event Record + if(settings.extractAA03) { + vsaRecords.push_back(std::make_shared(bytes + bytesOffset)); + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x04u: // Partition Info Record + if(settings.extractAA04) { + vsaRecords.push_back(std::make_shared(bytes + bytesOffset)); + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x05u: // Application Error Record + if(settings.extractAA05) { + vsaRecords.push_back(std::make_shared(bytes + bytesOffset)); + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x06u: // Debug/Internal + if(settings.extractAA06) { + vsaRecords.push_back(std::make_shared(bytes + bytesOffset)); + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x07u: // Debug/Internal + if(settings.extractAA07) { + vsaRecords.push_back(std::make_shared(bytes + bytesOffset)); + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x08u: // Buffer Info Record + if(settings.extractAA08) { + vsaRecords.push_back(std::make_shared(bytes + bytesOffset)); + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x09u: // Device Info Record + if(settings.extractAA09) { + vsaRecords.push_back(std::make_shared(bytes + bytesOffset)); + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x0Au: // Logger Info Configuration (Deprecated) + hasDeprecatedRecords = true; + bytesOffset += VSA::StandardRecordSize; + break; + case 0x0Bu: // Message Data + if(settings.extractAA0B) { + auto record = std::make_shared(bytes + bytesOffset); + vsaRecords.push_back(record); + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x0Cu: // PCM Audio Data + if(settings.extractAA0C) { + vsaRecords.push_back(std::make_shared(bytes + bytesOffset)); + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x0Du: // Message Data (Extended) + if(settings.extractAA0D) { + if(!handleExtendedRecord(bytes, bytesOffset, VSA::Type::AA0D)) { + return false; + } + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x0Eu: // Message Data (Extended) + if(settings.extractAA0E) { + if(!handleExtendedRecord(bytes, bytesOffset, VSA::Type::AA0E)) { + return false; + } + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x0Fu: // Message Data (Extended) + if(settings.extractAA0F) { + if(!handleExtendedRecord(bytes, bytesOffset, VSA::Type::AA0F)) { + return false; + } + } + bytesOffset += VSA::StandardRecordSize; + break; + case 0x6Au: // Logger Configuration Backup + if(bytesOffset + Disk::SectorSize <= arrLen) { + if(settings.extractAA6A) { + vsaRecords.push_back(std::make_shared(bytes + bytesOffset)); + } + } + bytesOffset += Disk::SectorSize; + break; + default: + // Unhandled VSA Record Type + return false; + break; + } + } + return true; +} + +bool VSAParser::handleExtendedRecord(uint8_t* const bytes, uint64_t& bytesOffset, VSA::Type type) +{ + // Gather info about the extended record sequence of the record contained in bytes + std::shared_ptr first; + uint16_t seqNum; + ExtendedMessageState::ExtendedRecordSeqInfo* seqInfo; + uint32_t runningChecksum = 0; + switch(type) { + case VSA::Type::AA0D: + first = std::make_shared(bytes + bytesOffset, runningChecksum); + seqNum = first->getSequenceNum(); + seqInfo = &state.vsa0DSeqInfo[seqNum]; + break; + case VSA::Type::AA0E: + first = std::make_shared(bytes + bytesOffset, runningChecksum); + seqNum = first->getSequenceNum(); + seqInfo = &state.vsa0ESeqInfo[seqNum]; + break; + case VSA::Type::AA0F: + first = std::make_shared(bytes + bytesOffset, runningChecksum); + seqNum = first->getSequenceNum(); + seqInfo = &state.vsa0FSeqInfo[seqNum]; + break; + default: + return false; // Invalid type was passed + } + + if(seqInfo->nextIndex == 0 && seqInfo->records.size() == 0) { // This is the first record in the sequence + if(first->getIndex() != 0) { + seqInfo->clear(); + report(APIEvent::Type::VSAExtendedMessageError, APIEvent::Severity::EventWarning); + return true; // This is not actually the first record + } + seqInfo->records.push_back(first); + seqInfo->totalRecordCount = first->getRecordCount(); + seqInfo->nextIndex++; + seqInfo->runningChecksum = runningChecksum; + } else if(seqInfo->nextIndex < seqInfo->totalRecordCount && seqInfo->records.size() > 0) { // Consecutive Record + std::shared_ptr consecutive; + bool isLast = seqInfo->nextIndex == seqInfo->totalRecordCount - 1; + + // Construct the consecutive record from bytes + switch(type) { + case VSA::Type::AA0D: + consecutive = std::make_shared( + bytes + bytesOffset, + seqInfo->runningChecksum, + std::dynamic_pointer_cast(seqInfo->records[0]), + isLast + ); + break; + case VSA::Type::AA0E: + consecutive = std::make_shared( + bytes + bytesOffset, + seqInfo->runningChecksum, + std::dynamic_pointer_cast(seqInfo->records[0]), + isLast + ); + break; + case VSA::Type::AA0F: + consecutive = std::make_shared( + bytes + bytesOffset, + seqInfo->runningChecksum, + std::dynamic_pointer_cast(seqInfo->records[0]), + isLast + ); + break; + default: + return false; + } + + if(consecutive->getIndex() == seqInfo->nextIndex && consecutive->getSequenceNum() == seqNum) { // This record is valid in the sequence + seqInfo->records.push_back(consecutive); + seqInfo->nextIndex++; + } else { // Sequence is out of order/invalid + // Throw away incomplete sequence and report warning + seqInfo->clear(); + report(APIEvent::Type::VSAExtendedMessageError, APIEvent::Severity::EventWarning); + // Save data for new sequence + if(first->getIndex() == 0) { + seqInfo->records.push_back(first); + seqInfo->totalRecordCount = first->getRecordCount(); + seqInfo->nextIndex++; + seqInfo->runningChecksum = runningChecksum; + } + return true; + } + if(seqInfo->nextIndex == seqInfo->totalRecordCount) { // This is the last record in the sequence + if(consecutive->getChecksumFailed()) { + // Fail out if checksum fails + seqInfo->clear(); + return false; + } + vsaRecords.insert(vsaRecords.end(), seqInfo->records.begin(), seqInfo->records.end()); + seqInfo->clear(); + } + } else { + return false; // Undefined behavior + } + return true; +} + +VSAParser::RecordParseStatus VSAParser::getRecordFromBytes(uint8_t* const bytes, size_t arrLen, std::shared_ptr& record) +{ + record = nullptr; + if(arrLen < VSA::StandardRecordSize) { + // Not enough bytes + return VSAParser::RecordParseStatus::InsufficientData; + } else if(bytes[0] != 0xAAu) { + return VSAParser::RecordParseStatus::NotARecordStart; + } else { + switch(bytes[1]) { + case 0x00u: // Pad Record + return VSAParser::RecordParseStatus::Pad; + case 0x01u: // Message Data (Deprecated) + return VSAParser::RecordParseStatus::Deprecated; + case 0x02u: // Logdata Record + if(settings.extractAA02) { + record = std::make_shared(bytes); + return VSAParser::RecordParseStatus::Success; + } + break; + case 0x03u: // Event Record + if(settings.extractAA03) { + record = std::make_shared(bytes); + return VSAParser::RecordParseStatus::Success; + } + break; + case 0x04u: // Partition Info Record + if(settings.extractAA04) { + record = std::make_shared(bytes); + return VSAParser::RecordParseStatus::Success; + } + break; + case 0x05u: // Application Error Record + if(settings.extractAA05) { + record = std::make_shared(bytes); + return VSAParser::RecordParseStatus::Success; + } + break; + case 0x06u: // Debug/Internal + if(settings.extractAA06) { + record = std::make_shared(bytes); + return VSAParser::RecordParseStatus::Success; + } + break; + case 0x07u: // Debug/Internal + if(settings.extractAA07) { + record = std::make_shared(bytes); + return VSAParser::RecordParseStatus::Success; + } + break; + case 0x08u: // Buffer Info Record + if(settings.extractAA08) { + record = std::make_shared(bytes); + return VSAParser::RecordParseStatus::Success; + } + break; + case 0x09u: // Device Info Record + if(settings.extractAA09) { + record = std::make_shared(bytes); + return VSAParser::RecordParseStatus::Success; + } + break; + case 0x0Au: + return VSAParser::RecordParseStatus::Deprecated; + case 0x0Bu: // Message Data + if(settings.extractAA0B) { + record = std::make_shared(bytes); + return VSAParser::RecordParseStatus::Success; + } + break; + case 0x0Cu: // PCM Audio Data + if(settings.extractAA0C) { + record = std::make_shared(bytes); + return VSAParser::RecordParseStatus::Success; + } + break; + case 0x0Du: // Message Data (Extended) + if(settings.extractAA0D) { + uint32_t payloadChecksum = 0; + const auto& vsa = std::make_shared(bytes, payloadChecksum); + record = vsa; + if(vsa->getIndex() == 0) { + return VSAParser::RecordParseStatus::Success; + } + // This returns the consecutive record as a first record + return VSAParser::RecordParseStatus::ConsecutiveExtended; + } + break; + case 0x0Eu: // Message Data (Extended) + if(settings.extractAA0E) { + uint32_t payloadChecksum = 0; + const auto& vsa = std::make_shared(bytes, payloadChecksum); + record = vsa; + if(vsa->getIndex() == 0) { + return VSAParser::RecordParseStatus::Success; + } + // This returns the consecutive record as a first record + return VSAParser::RecordParseStatus::ConsecutiveExtended; + } + break; + case 0x0Fu: // Message Data (Extended) + if(settings.extractAA0F) { + uint32_t payloadChecksum = 0; + const auto& vsa = std::make_shared(bytes, payloadChecksum); + record = vsa; + if(vsa->getIndex() == 0) { + return VSAParser::RecordParseStatus::Success; + } + // This returns the consecutive record as a first record + return VSAParser::RecordParseStatus::ConsecutiveExtended; + } + break; + case 0x6Au: // Logger Configuration Backup + if(settings.extractAA6A) { + if(arrLen < Disk::SectorSize) { + return VSAParser::RecordParseStatus::InsufficientData; + } + record = std::make_shared(bytes); + return VSAParser::RecordParseStatus::Success; + } + break; + default: + // Unhandled VSA Record Type + return VSAParser::RecordParseStatus::UnknownRecordType; + break; + } + } + return VSAParser::RecordParseStatus::FilteredOut; +} + +void VSAParser::clearParseState() +{ + for(size_t i = 0; i < state.vsa0DSeqInfo.size(); i++) { + state.vsa0DSeqInfo[i].clear(); + } + for(size_t i = 0; i < state.vsa0ESeqInfo.size(); i++) { + state.vsa0ESeqInfo[i].clear(); + } + for(size_t i = 0; i < state.vsa0DSeqInfo.size(); i++) { + state.vsa0ESeqInfo[i].clear(); + } +} + +bool VSAParser::extractMessagePackets(std::vector>& packets) +{ + if(settings != Settings::messageRecords()) { + report(APIEvent::Type::ParameterOutOfRange, APIEvent::Severity::Error); + return false; // We do not have exclusively message records + } + std::shared_ptr packet; + bool activeExtendedMessage = false; + VSA::Type previousRecordType = VSA::Type::Invalid; + for(const auto& record : vsaRecords) { + VSA::Type activeRecordType = record->getType(); + switch(activeRecordType) { + // Handle standard message records + case VSA::Type::AA0B: { + if(activeExtendedMessage) { + // Non-terminated extended message record + // There was a failure/unexpected behavior in the parsing process + report(APIEvent::Type::VSAExtendedMessageError, APIEvent::Severity::Error); + return false; + } + + std::shared_ptr messageRecord = std::dynamic_pointer_cast(record); + if(!settings.messageFilter || messageRecord->filter(settings.messageFilter)) { + packet = messageRecord->getPacket(); + packets.push_back(packet); + } + packet = nullptr; + break; + } + + // Handle extended message records + case VSA::Type::AA0D: + case VSA::Type::AA0E: + case VSA::Type::AA0F: { + std::shared_ptr extendedMessageRecord = std::dynamic_pointer_cast(record); + + if(!activeExtendedMessage) { // Start new extended message packet + packet = extendedMessageRecord->getPacket(); + activeExtendedMessage = true; + previousRecordType = extendedMessageRecord->getType(); + } else if(previousRecordType == activeRecordType) { // Continue existing extended message packet + extendedMessageRecord->appendPacket(packet); + if(extendedMessageRecord->getRecordCount() == static_cast(extendedMessageRecord->getIndex() + 1)) { // Last record in sequence + if(!settings.messageFilter || extendedMessageRecord->filter(settings.messageFilter)) { + VSAExtendedMessage::truncatePacket(packet); + packets.push_back(packet); + } + activeExtendedMessage = false; + packet = nullptr; + previousRecordType = activeRecordType; + } + } else { + // Non-terminated extended message record + // There was a failure/unexpected behavior in the parsing process + activeExtendedMessage = false; + packet = nullptr; + previousRecordType = VSA::Type::Invalid; + report(APIEvent::Type::VSAOtherError, APIEvent::Severity::Error); + return false; + } + break; + } + + default: + // Non-message record discovered + report(APIEvent::Type::VSAOtherError, APIEvent::Severity::Error); + return false; + } + } + vsaRecords.clear(); + return true; +} \ No newline at end of file diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 8ca17d5..a127c94 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -8,6 +8,7 @@ option(LIBICSNEO_BUILD_CPP_LIN_EXAMPLE "Build the LIN example." ON) option(LIBICSNEO_BUILD_CPP_LIVEDATA_EXAMPLE "Build the Live Data example." ON) option(LIBICSNEO_BUILD_CPP_COREMINI_EXAMPLE "Build the Coremini example." ON) option(LIBICSNEO_BUILD_CPP_MDIO_EXAMPLE "Build the MDIO example." ON) +option(LIBICSNEO_BUILD_CPP_VSA_EXAMPLE "Build the VSA example." ON) # Disabled until we properly build these in-tree # option(LIBICSNEO_BUILD_CSHARP_INTERACTIVE_EXAMPLE "Build the command-line interactive C# example." OFF) @@ -53,6 +54,10 @@ if(LIBICSNEO_BUILD_CPP_MDIO_EXAMPLE) add_subdirectory(cpp/mdio) endif() +if(LIBICSNEO_BUILD_CPP_VSA_EXAMPLE) + add_subdirectory(cpp/vsa) +endif() + # if(LIBICSNEO_BUILD_CSHARP_INTERACTIVE_EXAMPLE) # add_subdirectory(csharp) # endif() diff --git a/examples/cpp/vsa/CMakeLists.txt b/examples/cpp/vsa/CMakeLists.txt new file mode 100644 index 0000000..b7d82fd --- /dev/null +++ b/examples/cpp/vsa/CMakeLists.txt @@ -0,0 +1,27 @@ +cmake_minimum_required(VERSION 3.2) +project(libicsneocpp-vsa-example VERSION 0.1.0) + +set(CMAKE_CXX_STANDARD_REQUIRED 11) + +include(GNUInstallDirs) + +# Add an include directory like so if desired +#include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) + +# Enable Warnings +if(MSVC) + # Force to always compile with W4 + if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]") + string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") + endif() +else() #if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-switch -Wno-unknown-pragmas") +endif() + +# Add libicsneo, usually a git submodule within your project works well +#add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../third-party/libicsneo ${CMAKE_CURRENT_BINARY_DIR}/third-party/libicsneo) + +add_executable(libicsneocpp-vsa-example src/VSAExample.cpp) +target_link_libraries(libicsneocpp-vsa-example icsneocpp) \ No newline at end of file diff --git a/examples/cpp/vsa/src/VSAExample.cpp b/examples/cpp/vsa/src/VSAExample.cpp new file mode 100644 index 0000000..c521080 --- /dev/null +++ b/examples/cpp/vsa/src/VSAExample.cpp @@ -0,0 +1,94 @@ +#include +#include + +#include "icsneo/icsneocpp.h" + +void onEvent(std::shared_ptr event) { + std::cout << event->describe() << std::endl; +} + +int main(int argc, char* argv[]) { + if(argc != 2) { + std::cout << "Usage: libicsneocpp-vsa-example " << std::endl; + return -1; + } + + std::string deviceSerial = argv[1]; + + // register an event callback so we can see all logged events + icsneo::EventManager::GetInstance().addEventCallback(icsneo::EventCallback(onEvent)); + + auto devices = icsneo::FindAllDevices(); + if(devices.empty()) { + std::cout << "error: no devices found" << std::endl; + return -1; + } + + std::shared_ptr device; + for(auto&& d : devices) { + if(deviceSerial == d->getSerial()) { + device = d; + } + } + if(!device) { + std::cout << "error: failed to find a device with serial number: " << deviceSerial << std::endl; + return -1; + } + + std::cout << "info: found " << device->describe() << std::endl; + + if(!device->open()) { + std::cout << "error: unable to open device" << std::endl; + } + + device->stopScript(); + + uint64_t firstOffset; + std::shared_ptr firstRecord; + if(!device->findFirstVSARecord(firstOffset, firstRecord)) { + std::cout << "error: unable to find first VSA record" << std::endl; + return -1; + } + std::cout << "info: found first VSA record at " << firstOffset << std::endl; + + uint64_t origLastOffset; + std::shared_ptr origLastRecord; + if(!device->findLastVSARecord(origLastOffset, origLastRecord)) { + std::cout << "error: unable to find last VSA record" << std::endl; + return -1; + } + std::cout << "info: found last VSA record at " << origLastOffset << std::endl; + + uint64_t canFrameCount = 0; + uint64_t ethFrameCount = 0; + device->addMessageCallback(std::make_shared([&](std::shared_ptr msg) { + if(msg->type != icsneo::Message::Type::Frame) { + return; + } + const auto frame = std::static_pointer_cast(msg); + if(frame->network.getType() == icsneo::Network::Type::CAN) { + ++canFrameCount; + } else if(frame->network.getType() == icsneo::Network::Type::Ethernet) { + ++ethFrameCount; + } + })); + icsneo::VSAExtractionSettings settings; + { + auto& filter = settings.filters.emplace_back(); + filter.readRange.first = origLastRecord->getTimestampICSClock() - std::chrono::seconds(20); + filter.readRange.second = origLastRecord->getTimestampICSClock() - std::chrono::seconds(10); + } + { + auto& filter = settings.filters.emplace_back(); + filter.readRange.first = firstRecord->getTimestampICSClock() + std::chrono::seconds(0); + filter.readRange.second = firstRecord->getTimestampICSClock() + std::chrono::seconds(10); + } + std::cout << "info: reading two blocks of VSA, 10s from the start and 10s from the end..." << std::endl; + if(!device->readVSA(settings)) { + std::cout << "error: unable to read VSA" << std::endl; + return -1; + } + std::cout << "info: processed " << canFrameCount << " CAN frames and " << ethFrameCount << " Ethernet frames" << std::endl; + + return 0; +} \ No newline at end of file diff --git a/include/icsneo/api/event.h b/include/icsneo/api/event.h index 5e40b79..416da9c 100644 --- a/include/icsneo/api/event.h +++ b/include/icsneo/api/event.h @@ -158,6 +158,15 @@ public: FTIncorrectDevicePath = FTOK + 31, FTOtherError = FTOK + 32, + // VSA + VSABufferCorrupted = 0x5000, + VSATimestampNotFound = VSABufferCorrupted + 1, + VSABufferFormatError = VSABufferCorrupted + 2, + VSAMaxReadAttemptsReached = VSABufferCorrupted + 3, + VSAByteParseFailure = VSABufferCorrupted + 4, + VSAExtendedMessageError = VSABufferCorrupted + 5, + VSAOtherError = VSABufferCorrupted + 6, + NoErrorFound = 0xFFFFFFFD, TooManyEvents = 0xFFFFFFFE, Unknown = 0xFFFFFFFF diff --git a/include/icsneo/communication/communication.h b/include/icsneo/communication/communication.h index 65e1d5d..7d7fc6c 100644 --- a/include/icsneo/communication/communication.h +++ b/include/icsneo/communication/communication.h @@ -76,6 +76,8 @@ public: const std::shared_ptr& f = {}, std::chrono::milliseconds timeout = std::chrono::milliseconds(50)); + void dispatchMessage(const std::shared_ptr& msg); + std::function()> makeConfiguredPacketizer; std::unique_ptr packetizer; std::unique_ptr encoder; @@ -93,7 +95,6 @@ protected: std::mutex redirectingReadMutex; // Don't allow read to be disabled while in the redirectionFn std::mutex syncMessageMutex; - void dispatchMessage(const std::shared_ptr& msg); void handleInput(Packetizer& p, std::vector& readBytes); private: diff --git a/include/icsneo/communication/message/filter/messagefilter.h b/include/icsneo/communication/message/filter/messagefilter.h index e97fbbc..9aaea0f 100644 --- a/include/icsneo/communication/message/filter/messagefilter.h +++ b/include/icsneo/communication/message/filter/messagefilter.h @@ -29,10 +29,10 @@ public: if(message->type == Message::Type::Frame || message->type == Message::Type::Main51 || message->type == Message::Type::RawMessage || message->type == Message::Type::ReadSettings) { - RawMessage& frame = *static_cast(message.get()); - if(!matchNetworkType(frame.network.getType())) + const auto frame = std::dynamic_pointer_cast(message); + if(!matchNetworkType(frame->network.getType())) return false; - if(!matchNetID(frame.network.getNetID())) + if(!matchNetID(frame->network.getNetID())) return false; } else if (netid != Network::NetID::Any || networkType != Network::Type::Any) { return false; // Filtering on a NetID or Type, but this message doesn't have one diff --git a/include/icsneo/device/device.h b/include/icsneo/device/device.h index 7aecc83..8cf5dc6 100644 --- a/include/icsneo/device/device.h +++ b/include/icsneo/device/device.h @@ -43,6 +43,8 @@ #include "icsneo/communication/message/ethphymessage.h" #include "icsneo/third-party/concurrentqueue/concurrentqueue.h" #include "icsneo/platform/nodiscard.h" +#include "icsneo/disk/vsa/vsa.h" +#include "icsneo/disk/vsa/vsaparser.h" #define ICSNEO_FINDABLE_DEVICE_BASE(className, type) \ static constexpr DeviceType::Enum DEVICE_TYPE = type; \ @@ -581,6 +583,101 @@ public: bool unsubscribeLiveData(const LiveDataHandle& handle); bool clearAllLiveData(); + // VSA Read functions + + /** + * Read VSA message records from disk and dispatch the messages via Communication object. Default behavior excludes + * records older than the current CoreMini script and performs a full disk dump of other records. The CoreMini script is + * also stopped by default. + * + * @param extractionSettings Contains filters and other advanced settings for extraction process + * + * @return Returns false if there were failures during the read or parse processes or issues with record formatting, else true + */ + bool readVSA(const VSAExtractionSettings& extractionSettings = VSAExtractionSettings()); + + /** + * Determines important metadata about VSA record storage on the disk. Terminates at first failed attempt to retrieve information + * + * @param metadata The metadata object to store the probed information into + * @param extractionSettings The settings for this extraction of VSA data + * + * @return True if all metadata information was successfully found + */ + bool probeVSA(VSAMetadata& metadata, const VSAExtractionSettings& extractionSettings); + + /** + * Find the first VSA record chronologically from ring buffer in the VSA log file on disk + * + * @param firstOffset Variable used to pass out offset of first record on the disk + * @param firstRecord Variable used to pass out the first record in the buffer + * @param extractionSettings The settings for this extraction of VSA data + * @param optMetadata Metadata about the current state of the VSA log file + * (Must include valid CoreMini timestamp, disk size, and isOverlapped values) + * + * @return True if the first record was found successfully + */ + bool findFirstVSARecord(uint64_t& firstOffset, std::shared_ptr& firstRecord, + const VSAExtractionSettings& extractionSettings = VSAExtractionSettings(), std::optional optMetadata = std::nullopt); + + /** + * Find the last record chronologically from ring buffer in the VSA log file with a valid timestamp + * + * @param lastOffset Variable used to pass out the offset of the last record on the disk + * @param lastRecord Variable used to pass out the last record with a valid timestamp + * @param extractionSettings The settings for this extraction of VSA data + * @param optMetadata Metadata about the current state of the VSA log file + * (Must include valid CoreMini timestamp, disk size, and isOverlapped values) + * + * @return True if the last record was found successfully + */ + bool findLastVSARecord(uint64_t& lastOffset, std::shared_ptr& lastRecord, + const VSAExtractionSettings& extractionSettings = VSAExtractionSettings(), std::optional optMetadata = std::nullopt); + + /** + * Find the closest VSA record to the desired time_point + * + * @param point The desired time_point of the record + * @param vsaOffset Variable used to pass out offset of record closest to the desired time_point + * @param record Variable used to pass out the record closest to the desired time_point + * @param extractionSettings Settings for this extraction of VSA data + * @param optMetadata Optional param to include metadata about the VSA log file on disk + * + * @return Pair containing the location of the record closest to the desired time_point (in bytes from the beginning of VSA log file) and the record itself + */ + bool findVSAOffsetFromTimepoint( + ICSClock::time_point point, uint64_t& vsaOffset, std::shared_ptr& record, const VSAExtractionSettings& extractionSettings = VSAExtractionSettings(), + std::optional optMetadata = std::nullopt); + + /** + * Parse VSA message records with the given filter and dispatch them with this device's com channel + * + * @param metadata Important information about the VSA logfile (including first record location) + * @param extractionSettings Settings for this extraction of VSA data + * @param filter Struct used to determine which bytes to read and to filter out undesired VSA records + * + * @return True if there were no failures reading from disk, parsing VSA records, or dispatching VSA records + */ + bool parseVSA( + VSAMetadata& metadata, const VSAExtractionSettings& extractionSettings = VSAExtractionSettings(), + const VSAMessageReadFilter& filter = VSAMessageReadFilter()); + + /** + * Wrapper function for Device::readLogicalDisk(pos, into, amount, ...) that handles the VSA record ring buffer. + * Handles pos that is before the VSA::RecordStartOffset or is larger than the diskSize. + * Sets amount to maximum size of ring buffer if given amount is too large. + * + * @param pos Position to start read from in relation to VSA file start + * @param into The buffer to read bytes into from the disk + * @param amount The number of bytes to read into the buffer + * @param metadata Optional metadata param (used to determine disk size and if disk is overlapped) + * + * @return Returns value of return from readLogicalDisk with the given inputs + */ + std::optional vsaReadLogicalDisk( + uint64_t pos, uint8_t* into, uint64_t amount, std::optional metadata = std::nullopt + ); + protected: bool online = false; int messagePollingCallbackID = 0; @@ -758,6 +855,96 @@ private: void scriptStatusThreadBody(); void stopScriptStatusThreadIfNecessary(std::unique_lock lk); + // VSA Read functions + + /** + * Read the timestamp from disk of the VSA record stored at pos. If the timestamp is unparsable, attempt to read from + * previous records up to minPos + * + * @param parser The parser that is used to create a VSA record from the given buffer + * @param buffer Vector of bytes that stores a sector from the disk + * @param pos The location that the buffer was read from + * @param minPos The leftmost (minimum) offset from the beginning of the VSA log file to attempt to read from + * @param optMetadata Optional param to include metadata about the VSA log file on disk + * + * @return The timestamp of the first valid record found at or before the given position + */ + std::optional getVSATimestampOrBefore(VSAParser& parser, std::vector& buffer, uint64_t pos, uint64_t minPos, + std::optional optMetadata = std::nullopt); + + /** + * Read the timestamp from disk of the VSA record stored at pos. If the timestamp is unparsable, attempt to read from + * previous records up to maxPos + * + * @param parser The parser that is used to create a VSA record from the given buffer + * @param buffer Vector of bytes that stores a sector that was previously read from the disk + * @param pos The location that data in the buffer was read from + * @param maxPos The rightmost (maximum) offset from the beginning of the VSA log file to attempt to read from + * @param optMetadata Optional param to include metadata about the VSA log file on disk + * + * @return The timestamp of the first valid record found at or after the given position + */ + std::optional getVSATimestampOrAfter(VSAParser& parser, std::vector& buffer, uint64_t pos, uint64_t maxPos, + std::optional optMetadata = std::nullopt); + + /** + * Iterate over VSA records and dispatch the messages contained within them. For extended message records, we concatenate the payloads of all of + * the records together before dispatching. Dispatch is performed by Communication::dispatchMessage(...). + * + * @param parser The parser that holds the VSAMessage records to be dispatched + * + * @return True if dispatching of records is successful without unhandled issues from record parse, else false + */ + bool dispatchVSAMessages(VSAParser& parser); + + /** + * Determine if the ring buffer for VSA records has filled entirely and looped to the beginning. + * + * @param optMetadata Optional metadata param with partial information about current state of VSA log file + * (Must contain valid disk size and CoreMini timestamp) + * + * @return True if the buffer has looped; Returns std::nullopt if unable to determine + */ + std::optional isVSAOverlapped(std::optional optMetadata = std::nullopt); + + /** + * Find the first extended message record in the sequence of the given extended message record by backtracking in the disk. + * Results are returned by reference (not through the return value) + * + * @param record The extended message record whose sequence for which to find the first record + * @param pos The position of the given record in the VSA log file (in bytes) + * @param parser Used to parse indices and sequence numbers from the extended message records + * @param metadata Optional param to include metadata about the VSA log file on disk + * + * @return True if the first extended message record in the sequence was successfully found + */ + bool findFirstExtendedVSAFromConsecutive(std::shared_ptr& record, uint64_t& pos, + VSAParser& parser, std::optional metadata = std::nullopt); + + /** + * Find the first record before the given position that contains a valid timestamp. Results are returned by reference. + * + * @param record The record from which to backtrack in the VSA buffer + * @param pos The position of the given record in the VSA log file (in bytes) + * @param parser Used to parse records from the VSA buffer + * + * @return True if a record with valid timestamp is found within a set number of reads + */ + bool findPreviousRecordWithTimestamp(std::shared_ptr& record, uint64_t& pos, VSAParser& parser); + + /** + * Get the creation timestamp of the CoreMini script from VSA log file + * + * @return Timestamp in 25 nanosecond ticks since January 1, 2007 + */ + std::optional getCoreMiniScriptTimestamp(); + + /** + * Get the size of the VSA file storage system from the CoreMini script + * + * @return The size of the vsa log files on the disk + */ + std::optional getVSADiskSize(); }; } diff --git a/include/icsneo/device/tree/radgigastar/radgigastar.h b/include/icsneo/device/tree/radgigastar/radgigastar.h index bded847..04dddbb 100644 --- a/include/icsneo/device/tree/radgigastar/radgigastar.h +++ b/include/icsneo/device/tree/radgigastar/radgigastar.h @@ -24,7 +24,7 @@ public: protected: RADGigastar(neodevice_t neodevice, const driver_factory_t& makeDriver) : Device(neodevice) { - initialize(makeDriver); + initialize(makeDriver); } void setupPacketizer(Packetizer& packetizer) override { diff --git a/include/icsneo/disk/vsa/vsa.h b/include/icsneo/disk/vsa/vsa.h new file mode 100644 index 0000000..b15d4de --- /dev/null +++ b/include/icsneo/disk/vsa/vsa.h @@ -0,0 +1,312 @@ +#ifndef __VSA_H__ +#define __VSA_H__ + +#ifdef __cplusplus + +#include "icsneo/communication/network.h" +#include "icsneo/communication/packet.h" +#include +#include +#include +#include "stdint.h" + +namespace icsneo { + +using CaptureBitfield = uint16_t; + +static constexpr uint64_t ICSEpochHoursSinceUnix = 13514 * 24; // Number of hours between the start of the Unix epoch and the start of the ICS epoch +static constexpr uint64_t UINT63_MAX = 0x7FFFFFFFFFFFFFFFu; + +/** + * Struct that meets Clock format requirements from STL Chrono library. + * Indicates time for for the ICS Epoch (January 1, 2007) in 25 nanosecond ticks. + */ +struct ICSClock { + using rep = uint64_t; // Type for tick count + using period = std::ratio_multiply, std::nano>; // Ratio of tick length to seconds (25 nanoseconds) + using duration = std::chrono::duration; // Type for duration in 25 nanosecond ticks + using time_point = std::chrono::time_point; // Type for a point in time with respect to ICSClock + + static constexpr bool is_steady = true; // This clock does not move backwards + + /** + * Get the time_point at the current time with respect to ICSClock + * + * @return Time point at the current time with respect to ICSClock + */ + static time_point now() noexcept + { + return time_point { std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) - + std::chrono::hours(ICSEpochHoursSinceUnix) }; + } +}; + +using Timestamp = ICSClock::time_point; // Point in time to start or stop read + +class VSA; + +/** + * Holds metadata for the VSA log file + */ +struct VSAMetadata { + uint64_t firstRecordLocation = UINT64_MAX; // Location of the record with lowest timestamp in ring buffer + std::shared_ptr firstRecord = nullptr; // The record with lowest timestamp + uint64_t lastRecordLocation = UINT64_MAX; // Location of the record with the highest timestamp in ring buffer + std::shared_ptr lastRecord = nullptr; // The record with the highest timestamp + uint64_t bufferEnd = UINT64_MAX; // One byte beyond the last byte of the sequence started from lastRecordLocation + uint64_t diskSize = 0; // The size of the vsa log file on the disk + bool isOverlapped = false; // Determines if VSA ring buffer has looped to beginning + uint64_t coreMiniTimestamp = UINT64_MAX; // Timestamp of the CoreMini message in 25 nanosecond ticks since January 1, 2007 +}; + +/** + * Struct used to exclude VSA message records from parse + */ +struct VSAMessageReadFilter { + CaptureBitfield captureBitfield = UINT16_MAX; // The capture from which to gather VSA message records. UINT16_MAX indicates 'all captures' + + // The range of timestamps to collect record data from + std::pair readRange = std::make_pair(Timestamp(ICSClock::duration(0x0ull)), Timestamp(ICSClock::duration(UINT64_MAX))); + + static constexpr Timestamp MinTimestamp = Timestamp(ICSClock::duration(0x0ull)); + static constexpr Timestamp MaxTimestamp = Timestamp(ICSClock::duration(UINT64_MAX)); +}; + +struct VSAExtractionSettings { + bool parseOldRecords = false; + bool stopCoreMini = true; + std::vector filters; +}; + +/** + * Abstract VSA base class to store VSA record data read from VSA log file on disk + */ +class VSA { +public: + static constexpr size_t StandardRecordSize = 32; // Size of most VSA records + static constexpr uint64_t RecordStartOffset = 0x06000000u; // Offset of VSA record ring buffer from start of VSA log file + + /** + * Convert the given time_point object to a timestamp in 25 nanosecond ticks since January 1, 2007 + * + * @return Timestamp of the given time_point in 25 nanosecond ticks since January 1, 2007 + */ + static uint64_t getICSTimestampFromTimepoint(const Timestamp& point) noexcept { return point.time_since_epoch().count(); } + + /** + * Enum to determine what type of record is underlying VSA parent class + */ + enum class Type : uint16_t { + AA00 = 0xAA00u, // Pad + AA01 = 0xAA01u, // Message Data (Deprecated) + AA02 = 0xAA02u, // 'Logdata' + AA03 = 0xAA03u, // Event + AA04 = 0xAA04u, // Partition Info + AA05 = 0xAA05u, // Application Error + AA06 = 0xAA06u, // Internal/Debug + AA07 = 0xAA07u, // Internal/Debug + AA08 = 0xAA08u, // Buffer Info + AA09 = 0xAA09u, // Device Info + AA0A = 0xAA0Au, // Logger Configuration Info (Deprecated) + AA0B = 0xAA0Bu, // Message Data + AA0C = 0xAA0Cu, // PCM Audio Data + AA0D = 0XAA0Du, // Message Data (Extended) + AA0E = 0xAA0Eu, // Message Data (Extended) + AA0F = 0xAA0Fu, // Message Data (Extended) + AA6A = 0xAA6Au, // Logger Configuration Backup (512 Bytes) + Invalid = UINT16_MAX // Used to indicate unset or unhandled VSA record types + }; + + /** + * Get the record type + * + * @return Type of record + */ + Type getType() const { return type; } + + /** + * Get the timestamp stored in this record + * + * @return The record's timestamp in 25 nanosecond ticks since January 1, 2007 + */ + virtual uint64_t getTimestamp() = 0; + + /** + * Determine whether this record has a valid timestamp. All invalid timestamps are set to the maximum value for a uint64_t. + * + * @return True if the timestamp is set to a valid number + */ + bool isTimestampValid() { return getTimestamp() != UINT64_MAX && !checksumFailed; } + + /** + * Determine if the checksum for this record failed + * + * @return True if the checksum does not pass + */ + bool getChecksumFailed() { return checksumFailed; } + + /** + * Get the timestamp of this record in C++ native std::chrono::time_point + * + * @return Timestamp of record as an std::chrono::time_point + */ + Timestamp getTimestampICSClock() + { + return Timestamp(std::chrono::duration_cast(std::chrono::nanoseconds(getTimestamp() * 25))); + } + +protected: + /** + * Used to construct a VSA record from child class + */ + VSA() {} + + /** + * Used to set the type of this record in child class constructors + * + * @param recordType The type of this record + */ + void setType(Type recordType) { this->type = recordType; } + + /** + * Set whether the checksum was passed for this record. This is called in each child class constructor. + * + * @param fail True if checksum did not pass, else false + */ + void setChecksumFailed(bool fail) { checksumFailed = fail; } + +private: + /** + * Performs checksum on data in specific record type. Calls VSA::setChecksumFailed(...) from child class. + * + * @param recordBytes Bytestream of record to perform checksum with + */ + virtual void doChecksum(uint8_t* recordBytes) = 0; + + Type type = Type::Invalid; // The type of this record + bool checksumFailed = false; // Determines if checksum failed +}; + +/** + * Interface class for handling common functionality of VSAMessage record types (AA0B, AA0D, AA0E, AA0F) + */ +class VSAMessage : public VSA { +public: + static constexpr size_t CoreMiniPayloadSize = 24; // Size of CoreMini message (payload) + + /** + * Construct a packet from the message payload and network + * + * @return Packet constructed from payload and network + */ + std::shared_ptr getPacket() const; + + /** + * Reserve enough memory to store a CoreMini message in the given packet + * + * @param packet The packet in which we are reserving memory for a message + */ + virtual void reservePacketData(std::shared_ptr& packet) const { packet->data.reserve(CoreMiniPayloadSize); } + + /** + * Determine whether to filter out this VSAMessage record during parsing + * + * @param filter The filter struct to check this message record against + * + * @return True if this message passes the given filter + */ + virtual bool filter(const std::shared_ptr filter) = 0; + +protected: + /** + * Constructor for normal instance of VSAMessage class + * + * @param messageBytes Bytestream that begins at the start of the message payload + * @param numBytes The number of bytes that the message payload contains + * @param networkId The CoreMini ID of the network for this message + */ + VSAMessage(uint8_t* const messageBytes, size_t numBytes, Network::CoreMini networkId = static_cast(UINT16_MAX)) + : VSA(), payload(messageBytes, messageBytes + numBytes), network(networkId) {} + + std::vector payload; // CoreMini message/payload of VSA record containing message data + Network network; // CoreMini network of this message +}; + +/** + * Interface class for handling common functionality of VSA Extended Message records (AA0D, AA0E, AA0F) + */ +class VSAExtendedMessage : public VSAMessage { +public: + static void truncatePacket(std::shared_ptr packet); + + /** + * Appends the payload for this message to the given packet. + * Also sets the network of the packet if unset (used primarily for AA0F records which do not contain the network in the first extended message record). + * + * @param packet The packet to append this record's payload to + */ + void appendPacket(std::shared_ptr packet) const; + + /** + * Get the total number of records for this extended message + * + * @return Total number of records that this message spans + */ + uint32_t getRecordCount() const { return totalRecordCount; } + + /** + * Get the index of this record in the extended message sequence + * + * @return The index of this record + */ + uint16_t getIndex() { return index; }; + + /** + * Get the numerical id of the sequence of extended message records this record is a part of + * + * @return The sequence number of this extended message record + */ + uint16_t getSequenceNum() { return sequenceNum; } + +protected: + /** + * Constructor for normal instance of VSAExtendedMessage + * + * @param messageBytes Bytestream that begins at the start of the message payload + * @param numBytes The length of the message payload in bytes + * @param networkId The CoreMini ID of the network for this message + */ + VSAExtendedMessage(uint8_t* const messageBytes, size_t numBytes, Network::CoreMini networkId = static_cast(UINT16_MAX)) + : VSAMessage(messageBytes, numBytes, networkId) {} + + /** + * Set the total number of records for this message + * + * @param recordCount Total number of records for this message + */ + void setRecordCount(uint32_t recordCount) { totalRecordCount = recordCount; } + + /** + * Set the index of this record + * + * @param recordIndex The index of this record in its extended message sequence + */ + void setIndex(uint16_t recordIndex) { this->index = recordIndex; } + + /** + * Set the sequence number of this record + * + * @param seq The id for the extended message sequence this record is a part of + */ + void setSequenceNum(uint16_t seq) { sequenceNum = seq; } + +private: + uint32_t totalRecordCount; // The total number of records for the extended message + uint16_t index; // The index of this record in its extended message sequence + uint16_t sequenceNum; // The id of the sequence of records this record is a part of +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa02.h b/include/icsneo/disk/vsa/vsa02.h new file mode 100644 index 0000000..2e019a5 --- /dev/null +++ b/include/icsneo/disk/vsa/vsa02.h @@ -0,0 +1,55 @@ +#ifndef __VSA02_H__ +#define __VSA02_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +namespace icsneo { + +/** + * Class that contains data for Logdata records + */ +class VSA02 : public VSA { +public: + /** + * Constructor that parses the given bytestream + * + * @param bytes Bystream that contains data for Logdata VSA records + */ + VSA02(uint8_t* const bytes); + + /** + * Get the timestamp for this record in 25 nanosecond ticks since January 1, 2007 + * + * @return The timestamp for this record in 25 nanosecond ticks since January 1, 2007 + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform the checksum for this record + * + * @param bytes Bystream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + uint16_t constantIndex; // Index into CoreMini binary where constant data for this record can be found + struct Flags { + bool hasMoreData : 1; // Set to true if there are further Logdata records expected to terminate this "chain" + uint8_t numSamples : 3; // Number of valid samples in samples + bool isAscii : 1; // Set to true if the processing code should treat samples as an ASCII string + bool prefixTime : 1; // Set to true if the function block step that created this record requested that the timestamp be prepended on the output + bool sample0IsHex : 1; // Set to true if the value in sample 0 should be written as hex + bool sample1IsHex : 1; // Set to true if the value in sample 1 shoudl be written as hex + } flags; // Series of flags for this record + uint8_t pieceCount; // Value of the rolling counter for this "chain" of logdata records + uint64_t timestamp; // Timestamp in 25 nanosecond ticks since January 1, 2007 + std::vector samples; // Data for this record that varies based on the above flags. Either 2 32.32 fixed point values or 16 byte ASCII string + uint16_t checksum; // The sum of the previous 15 words +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA02_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa03.h b/include/icsneo/disk/vsa/vsa03.h new file mode 100644 index 0000000..ea710ad --- /dev/null +++ b/include/icsneo/disk/vsa/vsa03.h @@ -0,0 +1,54 @@ +#ifndef __VSA03_H__ +#define __VSA03_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +namespace icsneo { + +/** + * Class used to store event + */ +class VSA03 : public VSA { +public: + /** + * Constructor that extracts data from the given bytestream + * + * @param bytes Bytestream to extract VSA record data from + */ + VSA03(uint8_t* const bytes); + + /** + * Get the timestamp for this record in 25 nanosecond ticks since January 1, 2007 + * + * @return The timestamp for this record in 25 nanosecond ticks since January 1, 2007 + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform checksum for this record + * + * @param bytes Bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + enum class EventType : uint16_t { + CaptureStarted = 0, + StorageReconnected = 3, + FileSystemBufferOverflow = 4, + LoggerWentToSleep = 5, + Internal = 7, + CaptureStopped = 8, + LoggerPowerEvent = 9 + } eventType; // Enumerated value indicating which type of event occurred + uint16_t eventData; // Information about the event that is dependent on eventType + uint64_t timestamp; // Timestamp of this record in 25 nanosecond ticks since January 1, 2007 + uint16_t checksum; // The sum of the previous 7 words +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA03_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa04.h b/include/icsneo/disk/vsa/vsa04.h new file mode 100644 index 0000000..2f4f4f4 --- /dev/null +++ b/include/icsneo/disk/vsa/vsa04.h @@ -0,0 +1,49 @@ +#ifndef __VSA04_H__ +#define __VSA04_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +namespace icsneo { + +/** + * Class that contains a partition info record + */ +class VSA04 : public VSA { +public: + /** + * Constructor that created partition info record from bytestream + * + * @param bytes Bytestream to create this record from + */ + VSA04(uint8_t* const bytes); + + /** + * Get the timestamp of this record in 25 nanosecond ticks since January 1, 2007 + * + * @return Timestamp of this record in 25 nanosecond ticks since January 1, 2007 + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform the checksum for this record + * + * @param bytes Bytestream to check against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + struct Flags { + bool invalidRequestDetected : 1; // Indicates if an invalid request was detected + uint16_t reserved : 15; // Empty flag bits + } flags; // Mostly empty field for flags + uint16_t partitionIndex; // The index of the partition containing this record + uint64_t timestamp; // Timestamp of this record in 25 nanosecond ticks since January 1, 2007 + uint16_t checksum; // Sum of the previous 7 words +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA04_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa05.h b/include/icsneo/disk/vsa/vsa05.h new file mode 100644 index 0000000..949c91a --- /dev/null +++ b/include/icsneo/disk/vsa/vsa05.h @@ -0,0 +1,79 @@ +#ifndef __VSA05_H__ +#define __VSA05_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +namespace icsneo { + +/** + * Class that holds data for application error records + */ +class VSA05 : public VSA { +public: + /** + * Constructor to convert bytestream into application error record + * + * @param bytes The bytestream containing the record data + */ + VSA05(uint8_t* const bytes); + + /** + * Get the timestamp for this record in 25 nanosecond ticks since January 1, 2007 + * + * @return The timestamp for this record in 25 nanosecond ticks since January 1, 2007 + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform the checksum on this record + * + * @param bytes Bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + enum class ErrorType : uint16_t { + NetworkReceiveBufferOverflow = 0, + NetworkTransmitBufferOverflow = 1, + NetworkTransmitReportBufferOverflow = 2, + PeripheralProcessorCommunicationError = 3, + NetworkPeripheralOverflow = 4, + CommunicationPacketChecksumError = 6, + CommunicationPacketDetectedMissingByte = 7, + FailedToApplySettingsToNetwork = 9, + EnabledNetworkCountExceedsLicenseCapability = 10, + NetworkNotEnabled = 11, + DetectedInvalidTimestamp = 12, + LoadedDefaultSettings = 13, + DeviceAttemptedUnsupportedOperation = 14, + TrasmitBufferFillExceededThreshold = 17, + TransmitRequestedOnInvalidNetwork = 18, + TransmitRequestedOnTransmitIncapableNetwork = 19, + TransmitRequestedWhileControllersInactive = 20, + FilterMatchesExceedLimit = 21, + EthernetPreemptionError = 22, + TransmitWhileControllerModeInvalid = 23, + FragmentedEthernetIPFrame = 25, + TransmitBufferUnderrun = 26, + ActiveCoolingFailureDetected = 27, + OvertemperatureConditionDetected = 28, + UndersizedEthernetFrame = 30, + OversizedEthernetFrame = 31, + SystemWatchdogEventOcurred = 32, + SystemClockFailureDetected = 33, + RecoveredFromSystemClockFailure = 34, + SystemResetFailedPeripheralComponent = 35, + FailedToInitializeLoggerDisk = 41, + AttemptedToApplyInvalidSettingsToNetwork = 42 + } errorType; // Enumerated value indicating the type of error that occurred + uint16_t errorNetwork; // When applicable, the enumerated network index that the error occurred on + uint64_t timestamp; // Timestamp of this record in 25 nanosecond ticks since January 1, 2007 + uint16_t checksum; // Sum of the previous 7 words +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA05_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa06.h b/include/icsneo/disk/vsa/vsa06.h new file mode 100644 index 0000000..ae0a625 --- /dev/null +++ b/include/icsneo/disk/vsa/vsa06.h @@ -0,0 +1,49 @@ +#ifndef __VSA06_H__ +#define __VSA06_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +#include + +namespace icsneo { + +/** + * Class that holds data for an internal/debug VSA record + */ +class VSA06 : public VSA { +public: + /** + * Constructor to convert bytestream into internal/debug record + * + * @param bytes Bytestream to parse into internal/debug record + */ + VSA06(uint8_t* const bytes); + + /** + * Get the timestamp for this record in 25 nanosecond ticks since January 1, 2007 + * + * @return The timestamp for this record in 25 nanosecond ticks since January 1, 2007 + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform the checksum on this record + * + * @param bytes Bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + std::vector savedSectors; // Unknown + uint16_t error; // Unknown + uint16_t savedSectorsHigh; // Unknown + uint64_t timestamp; // Timestamp for this record in 25 nanosecond ticks since January 1, 2007 + uint16_t checksum; // Sum of the previous 15 words +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA06_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa07.h b/include/icsneo/disk/vsa/vsa07.h new file mode 100644 index 0000000..2278ee4 --- /dev/null +++ b/include/icsneo/disk/vsa/vsa07.h @@ -0,0 +1,49 @@ +#ifndef __VSA07_H__ +#define __VSA07_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +#include + +namespace icsneo { + +/** + * Class that holds data for an internal/debug VSA record + */ +class VSA07 : public VSA { +public: + /** + * Constructor to convert bytestream into internal/debug record + * + * @param bytes Bytestream to parse into internal/debug record + */ + VSA07(uint8_t* const bytes); + + /** + * Get the timestamp for this record in 25 nanosecond ticks since January 1, 2007 + * + * @return The timestamp for this record in 25 nanosecond ticks since January 1, 2007 + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform the checksum on this record + * + * @param bytes Bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + uint32_t lastSector; // Unknown + uint32_t currentSector; // Unknown + std::vector reserved; // Unused bytes (12 bytes) + uint64_t timestamp; // Timestamp for this record in 25 nanosecond ticks since January 1, 2007 + uint16_t checksum; // Sum of the previous 15 words +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA07_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa08.h b/include/icsneo/disk/vsa/vsa08.h new file mode 100644 index 0000000..18c76a7 --- /dev/null +++ b/include/icsneo/disk/vsa/vsa08.h @@ -0,0 +1,46 @@ +#ifndef __VSA08_H__ +#define __VSA08_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +namespace icsneo { + +/** + * Class to store data for a buffer information record + */ +class VSA08 : public VSA { +public: + /** + * Constructor to convert a bytestream to a buffer info record + * + * @param bytes Bytestream to convert into a buffer record + */ + VSA08(uint8_t* const bytes); + + /** + * Get the timestamp for this record in 25 nanosecond ticks since January 1, 2007 + * + * @return The timestamp for this record in 25 nanosecond ticks since January 1, 2007 + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform the checksum on this record + * + * @param bytes Bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + std::vector troubleSramCount; // Unknown (4 bytes) + std::vector troubleSectors; // Unknown (16 bytes) + uint64_t timestamp; // Timestamp for this record in 25 nanosecond ticks since January 1, 2007 + uint16_t checksum; // Sum of the previous 15 words +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA08_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa09.h b/include/icsneo/disk/vsa/vsa09.h new file mode 100644 index 0000000..7aeff02 --- /dev/null +++ b/include/icsneo/disk/vsa/vsa09.h @@ -0,0 +1,63 @@ +#ifndef __VSA09_H__ +#define __VSA09_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +namespace icsneo { + +/** + * Class used to store data for a Device Information VSA Record + */ +class VSA09 : public VSA { +public: + /** + * Constructor to convert bytestream to Device Information VSA Record + * + * @param bytes Bytestream to convert into Device Information Record + */ + VSA09(uint8_t* const bytes); + + /** + * Get the timestamp for this record in 25 nanosecond ticks since January 1, 2007 + * + * @return The timestamp for this record in 25 nanosecond ticks since January 1, 2007 + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform the checksum on this record + * + * @param bytes Bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + uint32_t serialNumber; // Decimal representation of Base-36 serial number + uint8_t firmwareMajorVersion; // Major version of firmware (A for A.B version) + uint8_t firmwareMinorVersion; // Minor version of firmware (B for A.B version) + uint8_t manufactureMajorRevision; // Major version of manufacture revision (A for A.B revision) + uint8_t manufactureMinorRevision; // Minor version of manufacture revision (B for A.B revision) + uint8_t bootloaderMajorVersion; // Major version of bootloader (A for A.B version) + uint8_t bootloaderMinorVersion; // Minor version of bootloader (B for A.B version) + std::vector reserved0; // Unused bytes (6 bytes) + enum class HardwareID : uint8_t { + NeoVIRED = 0, + NeoVIFIRE = 1, + NeoVIION = 11, + RADGalaxy = 19, + RADMars = 29, + ValueLOG = 31, + NeoVIRED2FIRE3 = 33, + RADGigastar = 36 + } hardwareID; // Identifier for specific hardware device type + std::vector reserved1; // Unused bytes (3 bytes) + uint64_t timestamp; // Timestamp for this record in 25 nanosecond ticks since January 1, 2007 + uint16_t checksum; // Sum of the previous 15 words +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA09_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa0b.h b/include/icsneo/disk/vsa/vsa0b.h new file mode 100644 index 0000000..a32ea51 --- /dev/null +++ b/include/icsneo/disk/vsa/vsa0b.h @@ -0,0 +1,57 @@ +#ifndef __VSA0B_H__ +#define __VSA0B_H__ + +#ifdef __cplusplus + +#include +#include "icsneo/disk/vsa/vsa.h" + +namespace icsneo { + +/** + * Class that holds single-record message records + */ +class VSA0B : public VSAMessage { +public: + /** + * Constructor that reads message record data from bytestream + * + * @param bytes Bytestream to read message record data from + */ + VSA0B(uint8_t* const bytes); + + /** + * Determine whether to filter out this message record + * + * @param filter The filter to check this record against + * + * @return True if this record has passed the filter (i.e., is not being filtered out) + */ + bool filter(const std::shared_ptr filter) override; + + /** + * Get the timestamp of this record in 25 nanosecond ticks since January 1, 2007 + * + * @return Timestamp of this record in 25 nanosecond ticks since January 1, 2007 + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform a checksum on this record + * + * @param bytes Bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + uint16_t captureBitfield; // The capture that this record is a part of + uint8_t reserved; // Unused bytes + uint16_t checksum; // Sum of the previous 15 half words + + uint64_t timestamp; // Timestamp of this record in 25 nanosecond ticks since January 1, 2007 (extracted from CoreMini message payload) +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA0B_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa0c.h b/include/icsneo/disk/vsa/vsa0c.h new file mode 100644 index 0000000..116b44b --- /dev/null +++ b/include/icsneo/disk/vsa/vsa0c.h @@ -0,0 +1,54 @@ +#ifndef __VSA0C_H__ +#define __VSA0C_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +#include + +namespace icsneo { + +/** + * Class used to hold data for a PCM Audio VSA Record + */ +class VSA0C : public VSA { +public: + /** + * Constructor to convert bytestream to PCM Audio Record + * + * @param bytes Bytestream to parse + */ + VSA0C(uint8_t* const bytes); + + /** + * Get the timestamp for this record in 25 nanosecond ticks since January 1, 2007 + * + * @return The timestamp for this record in 25 nanosecond ticks since January 1, 2007 + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform the checksum on this record + * + * @param bytes Bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + uint16_t captureBitfield; // Capture this record is a member of (Unused) + uint8_t audioPreamble; // Unknown + uint8_t audioHeader; // Unknown + std::vector pcmData; // Audio data payload (14 bytes) + uint64_t timestamp; // Timestamp of this record in 25 nanosecond ticks since January 1, 2007 + struct VNet { + uint16_t vNetSlot : 2; // Bits to identify VNet slot of this record + uint16_t reserved : 14; // Unused bits + } vNetBitfield; // Struct to ensure VNetSlot is only 2 bits + uint16_t checksum; // Sum of the previous 15 words +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA0C_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa0d.h b/include/icsneo/disk/vsa/vsa0d.h new file mode 100644 index 0000000..6ab2c3a --- /dev/null +++ b/include/icsneo/disk/vsa/vsa0d.h @@ -0,0 +1,134 @@ +#ifndef __VSA0D_H__ +#define __VSA0D_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +namespace icsneo { + +/** + * Base class for VSA0D extended message record types + */ +class VSA0D : public VSAExtendedMessage { +public: + /** + * Constructor for VSA0D parent class. Passes most information to VSAExtendedMessage constructor. + * + * @param bytes Bytestream to read record data from + * @param messageBytes Bytestream starting at the payload of this message record + * @param numBytes The length of the payload of this message record + * @param runningChecksum Checksum for the payload bytes of this sequence of extended message records + * @param networkId The CoreMini Network ID for this record + */ + VSA0D(uint8_t* const bytes, uint8_t* const messageBytes, size_t numBytes, uint32_t& runningChecksum, Network::CoreMini networkId = static_cast(0xFFFFu)); +}; + +/** + * Class holding data for the first record in a series of VSA0D extended message records + */ +class VSA0DFirst : public VSA0D { +public: + /** + * Constructor that parses first 32 bytes of bytestream into readable data. + * + * @param bytes Bytestream to parse VSA record from + */ + VSA0DFirst(uint8_t* const bytes, uint32_t& runningChecksum); + + /** + * Reserve memory in the packet data vector to store message data from this record and subsequent consecutive records. + * + * @param packet The packet to reserve memory in + */ + void reservePacketData(std::shared_ptr& packet) const override; + + /** + * Determine whether to filter out this message record + * + * @param filter The filter to check this record against + * + * @return True if the record passes the filter + */ + bool filter(const std::shared_ptr filter) override; + + /** + * Get the timestamp of this record. Timestamp indicates number of 25 nanosecond ticks since January 1, 2007. + * + * @return The timestamp of this record + */ + uint64_t getTimestamp() override { return timestamp; } + + /** + * Reorder bytes in the payload between this record and the second record in the sequence. The bytes are reordered to allow + * simple concatenation of payload bytes from records before creating and dispatching a packet. + * + * @param secondPayload Reference to the payload from the second record in the sequence + */ + void reorderPayload(std::vector& secondPayload); + +private: + /** + * Perform checksum on this record with the given bytestream + * + * @param bytes The bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + uint16_t captureBitfield; // The data capture this record is a part of + uint64_t timestamp; // The timestamp of this record in 25 nanosecond ticks since January 1, 2007 + struct VNet { + uint8_t vNetSlot : 2; // VNet bytes + uint8_t reserved : 6; // Unused bytes + } vNetInfo; // Struct used to indicate which bytes are actually used for VNetSlot + uint16_t checksum; // The sum of the previous 15 words +}; + +/** + * Class holding data for subsequent records in series of VSA0D extended message records + */ +class VSA0DConsecutive : public VSA0D { +public: + /** + * Constructor that parses first 32 bytes of VSA0D record + * @param bytes Bytestream to read VSA record data from + * @param first The first record in this series of VSA0D records + * @param isLastRecord Determines if this record is the last record in this series of extended message records + */ + VSA0DConsecutive(uint8_t* const bytes, uint32_t& runningChecksum, std::shared_ptr first, bool isLastRecord = false); + + /** + * Determine whether to filter out this message record. Utilizes the filter from the first record. + * + * @param filter The filter to check this record against + * + * @return True if this record passes the filter + */ + bool filter(const std::shared_ptr filter) override { return first ? first->filter(filter) : false; } + + /** + * Get the timestamp of this record in 25 nanosecond ticks since January 1, 2007. + * + * @return Timestamp in 25 nanosecond ticks since January 1, 2007. + */ + uint64_t getTimestamp() override { return first ? first->getTimestamp() : UINT64_MAX; } + +private: + /** + * Perform checksum on this record with the given bytestream + * + * @param bytes The bytestream to test the checksum against + */ + void doChecksum(uint8_t* bytes) override; + + uint32_t recordChecksum; // The checksum for this extended record sequence found in the last record (not used if not last record) + + uint32_t calculatedChecksum = 0; // Running checksum total for the extended record sequence of this record + + std::shared_ptr first = nullptr; // The first record in this extended message record series +}; + +} // namespace icsneo + +#endif //__cplusplus +#endif // __VSA0D_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa0e.h b/include/icsneo/disk/vsa/vsa0e.h new file mode 100644 index 0000000..b2c4495 --- /dev/null +++ b/include/icsneo/disk/vsa/vsa0e.h @@ -0,0 +1,130 @@ +#ifndef __VSA0E_H__ +#define __VSA0E_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +namespace icsneo { + +/** + * Base class for VSA0E extended message record types + */ +class VSA0E : public VSAExtendedMessage { +public: + /** + * Constructor for VSA0E parent class. Passes most information to VSAExtendedMessage constructor. + * + * @param bytes Bytestream to read record data from + * @param messageBytes Bytestream starting at the payload of this message record + * @param numBytes The length of the payload of this message record + * @param runningChecksum Checksum for the payload bytes of this sequence of extended message records + * @param networkId The CoreMini Network ID for this record + */ + VSA0E(uint8_t* const bytes, uint8_t* const messageBytes, size_t numBytes, uint32_t& runningChecksum, Network::CoreMini networkId = static_cast(0xFFFFu)); +}; + +/** + * Class holding data for the first record in a series of VSA0E extended message records + */ +class VSA0EFirst : public VSA0E { +public: + /** + * Constructor that parses first 32 bytes of bytestream into readable data. + * + * @param bytes Bytestream to parse VSA record from + */ + VSA0EFirst(uint8_t* const bytes, uint32_t& runningChecksum); + + /** + * Reserve memory in the packet data vector to store message data from this record and subsequent consecutive records. + * + * @param packet The packet to reserve memory in + */ + void reservePacketData(std::shared_ptr& packet) const override; + + /** + * Determine whether to filter out this message record + * + * @param filter The filter to check this record against + * + * @return True if the record passes the filter + */ + bool filter(const std::shared_ptr filter) override; + + /** + * Get the timestamp of this record. Timestamp indicates number of 25 nanosecond ticks since January 1, 2007. + * + * @return The timestamp of this record + */ + uint64_t getTimestamp() override { return timestamp; } + + /** + * Reorder bytes in the payload between this record and the second record in the sequence. The bytes are reordered to allow + * simple concatenation of payload bytes from records before creating and dispatching a packet. + * + * @param secondPayload Reference to the payload from the second record in the sequence + */ + void reorderPayload(std::vector& secondPayload); + +private: + /** + * Perform the checksum on this record + * + * @param bytes Bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + uint16_t captureBitfield; // The data capture this record is a part of + uint64_t timestamp; // Timestamp of this record in 25 nanosecond ticks since January 1, 2007 + uint16_t checksum; // Sum of the previous 15 words +}; + +/** + * Class holding data for subsequent records in series of VSA0E extended message records + */ +class VSA0EConsecutive : public VSA0E { +public: + /** + * Constructor that parses first 32 bytes of VSA0E record + * @param bytes Bytestream to read VSA record data from + * @param first The first record in this series of VSA0E records + * @param isLastRecord Determines if this record is the last record in this series of extended message records + */ + VSA0EConsecutive(uint8_t* const bytes, uint32_t& runningChecksum, std::shared_ptr first, bool isLastRecord = false); + + /** + * Determine whether to filter out this message record. Utilizes the filter from the first record. + * + * @param filter The filter to check this record against + * + * @return True if this record passes the filter + */ + bool filter(const std::shared_ptr filter) override { return first ? first->filter(filter) : false; } + + /** + * Get the timestamp of this record in 25 nanosecond ticks since January 1, 2007. + * + * @return Timestamp in 25 nanosecond ticks since January 1, 2007. + */ + uint64_t getTimestamp() override { return first ? first->getTimestamp() : UINT64_MAX; } + +private: + /** + * Perform checksum on this record with the given bytestream + * + * @param bytes The bytestream to test the checksum against + */ + void doChecksum(uint8_t* bytes) override; + + uint32_t recordChecksum; // The checksum for this extended record sequence found in the last record (not used if not last record) + + uint32_t calculatedChecksum; // Running checksum total for the extended record sequence of this record + + std::shared_ptr first = nullptr; // The first record in this series of extended message records +}; + +} // namespace icsneo + +#endif //__cplusplus +#endif // __VSA0E_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa0f.h b/include/icsneo/disk/vsa/vsa0f.h new file mode 100644 index 0000000..98f97a0 --- /dev/null +++ b/include/icsneo/disk/vsa/vsa0f.h @@ -0,0 +1,119 @@ +#ifndef __VSA0F_H__ +#define __VSA0F_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +namespace icsneo { + +/** + * Base class for VSA0F extended message record types + */ +class VSA0F : public VSAExtendedMessage { +public: + /** + * Standard constructor for VSA0F parent class. Passes most information to VSAExtendedMessage constructor. + * + * @param bytes Bytestream to read record data from + * @param messageBytes Bytestream starting at the payload of this message record + * @param numBytes The length of the payload of this message record + * @param runningChecksum Checksum for the payload bytes of this sequence of extended message records + * @param networkId The CoreMini Network ID for this record + */ + VSA0F(uint8_t* const bytes, uint8_t* const messageBytes, size_t numBytes, uint32_t& runningChecksum, Network::CoreMini networkId = static_cast(0xFFFFu)); +}; + +/** + * Class holding data for the first record in a series of VSA0F extended message records + */ +class VSA0FFirst : public VSA0F { +public: + /** + * Constructor that parses first 32 bytes of bytestream into readable data. + * + * @param bytes Bytestream to parse VSA record from + */ + VSA0FFirst(uint8_t* const bytes, uint32_t& runningChecksum); + + /** + * Reserve memory in the packet data vector to store message data from this record and subsequent consecutive records. + * + * @param packet The packet to reserve memory in + */ + void reservePacketData(std::shared_ptr& packet) const override; + + /** + * Determine whether to filter out this message record + * + * @param filter The filter to check this record against + * + * @return True if the record passes the filter + */ + bool filter(const std::shared_ptr filter) override; + + /** + * Get the timestamp of this record. Timestamp indicates number of 25 nanosecond ticks since January 1, 2007. + * + * @return The timestamp of this record + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform checksum on this record with the given bytestream + * + * @param bytes The bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + uint16_t captureBitfield; // The data capture this record is a part of + uint64_t timestamp; // The timestamp of this record in 25 nanosecond ticks since January 1, 2007 + uint16_t checksum; // The sum of the previous 9 words (Does not include message payload since payload bits follow the checksum) +}; +/** + * Class holding data for subsequent records in series of VSA0D extended message records + */ +class VSA0FConsecutive : public VSA0F { +public: + /** + * Constructor that parses first 32 bytes of VSA0F record + * @param bytes Bytestream to read VSA record data from + * @param first The first record in this series of VSA0F records + * @param isLastRecord Determines if this record is the last record in this series of extended message records + */ + VSA0FConsecutive(uint8_t* const bytes, uint32_t& runningChecksum, std::shared_ptr first, bool isLastRecord = false); + + /** + * Determine whether to filter out this message record. Utilizes the filter from the first record. + * + * @param filter The filter to check this record against + * + * @return True if this record passes the filter + */ + bool filter(const std::shared_ptr filter) override { return first ? first->filter(filter) : false; } + + /** + * Get the timestamp of this record in 25 nanosecond ticks since January 1, 2007. + * + * @return Timestamp in 25 nanosecond ticks since January 1, 2007. + */ + uint64_t getTimestamp() override { return first ? first->getTimestamp() : UINT64_MAX; } + +private: + /** + * Perform checksum on this record with the given bytestream + * + * @param bytes The bytestream to test the checksum against + */ + void doChecksum(uint8_t* bytes) override; + + uint32_t calculatedChecksum; // Running checksum total for the extended record sequence of this record + + std::shared_ptr first = nullptr; // The first record in this series of extended message records +}; + +} // namespace icsneo + +#endif //__cplusplus +#endif // __VSA0F_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsa6a.h b/include/icsneo/disk/vsa/vsa6a.h new file mode 100644 index 0000000..c658d07 --- /dev/null +++ b/include/icsneo/disk/vsa/vsa6a.h @@ -0,0 +1,49 @@ +#ifndef __VSA6A_H__ +#define __VSA6A_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" + +namespace icsneo { + +/** + * Class that contains data for a Logger Configuration Backup Record + */ +class VSA6A : public VSA { +public: + /** + * Constructor to convert a bytestream to a Logger Configuration Backup Record + * + * @param bytes Bytestream to convert to Logger Configuration Backup Record + */ + VSA6A(uint8_t* const bytes); + + /** + * Get the timestamp for this record in 25 nanosecond ticks since January 1, 2007 + * + * @return The timestamp for this record in 25 nanosecond ticks since January 1, 2007 + */ + uint64_t getTimestamp() override { return timestamp; } + +private: + /** + * Perform the checksum on this record + * + * @param bytes Bytestream to test against the checksum + */ + void doChecksum(uint8_t* bytes) override; + + uint32_t sequenceNum; // Unknown + uint32_t totalSectors; // Unknown + uint32_t reserved; // Unused bytes + uint64_t timestamp; // Timestamp of this record in 25 nanosecond ticks since January 1, 2007 + uint16_t timestampSum; // Sum of the bytes in this record's timestamp (previous 8 bytes) + std::vector data; // Payload data for this record (452 bytes) + uint32_t checksum; // Sum of the previous 452 bytes (bytes from data) +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSA6A_H__ \ No newline at end of file diff --git a/include/icsneo/disk/vsa/vsaparser.h b/include/icsneo/disk/vsa/vsaparser.h new file mode 100644 index 0000000..20ca726 --- /dev/null +++ b/include/icsneo/disk/vsa/vsaparser.h @@ -0,0 +1,234 @@ +#ifndef __VSAPARSER_H__ +#define __VSAPARSER_H__ + +#ifdef __cplusplus + +#include "icsneo/disk/vsa/vsa.h" +#include "icsneo/disk/vsa/vsa0d.h" +#include "icsneo/disk/vsa/vsa0e.h" +#include "icsneo/disk/vsa/vsa0f.h" +#include "icsneo/communication/message/message.h" +#include "icsneo/api/eventmanager.h" + +#include +#include + +namespace icsneo { + +/** + * Class used to parse VSA records from bytestreams + */ +class VSAParser { +public: + /** + * Struct that determines which types of VSA records to extract from disk + */ + struct Settings { + bool extractAA02 = true; // Extract Logdata Records + bool extractAA03 = true; // Extract Event Records + bool extractAA04 = true; // Extract Partition Info Records + bool extractAA05 = true; // Extract Application Error Records + bool extractAA06 = true; // Extract Debug/Internal 1 Records + bool extractAA07 = true; // Extract Debug/Internal 2 Records + bool extractAA08 = true; // Extract Buffer Info Records + bool extractAA09 = true; // Extract Device Info Records + bool extractAA0B = true; // Extract Message Records + bool extractAA0C = true; // Extract PCM Audio Records + bool extractAA0D = true; // Extract Extended Message 1 Records + bool extractAA0E = true; // Extract Extended Message 2 Records + bool extractAA0F = true; // Extract Extended Message 3 Records + bool extractAA6A = true; // Extract Logger Configuration Backup Records + + std::shared_ptr messageFilter = nullptr; // Used for post-read filtering of message records + + /** + * Static constructor for VSAParser::Settings that only extracts message records (AA0B, AA0D, AA0E, AA0F) + */ + static Settings messageRecords() { return { false, false, false, false, false, false, false, false, true, false, true, true, true, false }; } + + /** + * Operator overload for equivalency of VSAParser::Settings struct + * + * @param s The settings object to test against this settings object + * + * @return True if the extraction settings are the same. Does not check the filter + */ + bool operator==(const Settings& s) + { + return s.extractAA02 == this->extractAA02 && s.extractAA03 == this->extractAA03 && s.extractAA04 == this->extractAA04 && + s.extractAA05 == this->extractAA05 && s.extractAA06 == this->extractAA06 && s.extractAA07 == this->extractAA07 && + s.extractAA08 == this->extractAA08 && s.extractAA09 == this->extractAA09 && s.extractAA0B == this->extractAA0B && + s.extractAA0C == this->extractAA0C && s.extractAA0D == this->extractAA0D && s.extractAA0E == this->extractAA0E && + s.extractAA0F == this->extractAA0F && s.extractAA6A == this->extractAA6A; + } + + /** + * Operator overload for non-equivalency of VSAParser::Settings struct + * + * @param s The settings object to test against this settings object + * + * @return True if the extraction settings are not the same. Does not check the filter + */ + bool operator!=(const Settings& s) + { + return !(*this == s); + } + }; + + /** + * Enumerated values to determine status of attempt to parse out-of-context record from bytestream + */ + enum class RecordParseStatus : uint8_t { + NotARecordStart, // Indicates first byte was not of format required for VSA records + Pad, // This record is a pad record + Deprecated, // This record is deprecated + ConsecutiveExtended, // This is a consecutive extended message record (i.e., not the first record in an extended message sequence) + FilteredOut, // This record was filtered out due to the current Settings of the VSAParser + UnknownRecordType, // The second byte indicates a record type that is unknown/not handled + InsufficientData, // There were not enough bytes given to the parse call + Success // The record was successfully parsed + }; + + /** + * Constructor with default settings + * + * @param report Handler to report APIEvents + */ + VSAParser(const device_eventhandler_t& report) { this->report = report; } + + /** + * Constructor with non-default settings + * + * @param report Handler to report APIEvents + * @param settings The settings to use for this parser + */ + VSAParser(const device_eventhandler_t& report, const Settings& settings) + : settings(settings) { this->report = report; } + + /** + * Parse the given bytestream into VSA records and store them in vsaRecords. + * Non-terminated extended message record sequences are stored in a temporary buffer until they terminate. + * + * @param bytes Bytestream to parse VSA records from + * @param arrLen The number of bytes in the bytestream + * + * @return True if there was no failure or unhandled behavior during parse, else false + */ + bool parseBytes(uint8_t* const bytes, uint64_t arrLen); + + /** + * Get the last fully-parsed record in the parser + */ + std::shared_ptr& back() { return vsaRecords.back(); } + + /** + * Get the number of records contained within the parser + * + * @return Size of the vector of VSA records + */ + size_t size() { return vsaRecords.size(); } + + /** + * Determine if number of records contained within the parser is 0 + * + * @return True if the parser record container is empty + */ + bool empty() { return vsaRecords.empty(); } + + /** + * Clear all fully-parsed records from the parser. Does not affect non-terminated extended message records stored in buffers. + */ + void clearRecords() { vsaRecords.clear(); } + + /** + * Parse first record from the given bytestream. + * + * @param bytes The bytestream to read from + * @param arrLen Length of the bytestream + * @param record Variable to pass out the record if able to parse + * + * @return The status of the record parse + */ + RecordParseStatus getRecordFromBytes(uint8_t* const bytes, size_t arrLen, std::shared_ptr& record); + + /** + * Set a message filter for the Settings for this VSAParser + * + * @param filter The message filter to set for this VSAParser + */ + void setMessageFilter(const VSAMessageReadFilter& filter) { settings.messageFilter = std::make_shared(filter); } + + /** + * Remove the message filter for the Settings from this parser + */ + void clearMessageFilter() { settings.messageFilter = nullptr; } + + /** + * Clear all extended message buffers and parse states + */ + void clearParseState(); + + /** + * Extract all packets from fully-parsed VSA records and store them in the given buffer + * + * @param packets The vector in which to store the packets from fully-parsed records + * + * @return True if packets were successfully extracted + */ + bool extractMessagePackets(std::vector>& packets); + +private: + /** + * Holds the state of all possible extended message record sequences (most will be empty/null) + */ + struct ExtendedMessageState { + /** + * Holds the state of a single extended message record sequence + */ + struct ExtendedRecordSeqInfo { + /** + * Reset the state of this record sequence + */ + void clear() + { + nextIndex = 0; + totalRecordCount = 0; + runningChecksum = 0; + records.clear(); + records.shrink_to_fit(); + } + + uint16_t nextIndex = 0; // The next index to be parsed in this sequence + uint32_t totalRecordCount = 0; // The total number of records that are in this sequence + uint32_t runningChecksum = 0; // The running calculated checksum for this sequence + + std::vector> records; // All of the records in this sequence + }; + + std::array vsa0DSeqInfo; // Holds state for each possible sequence ID for VSA0D + std::array vsa0ESeqInfo; // Holds state for each possible sequence ID for VSA0E + std::array vsa0FSeqInfo; // Holds state for each possible sequence ID for VSA0F + }; + + /** + * Handle parsing of extended message records + * + * @param bytes The bytestream to parse the extended message record from + * @param bytesOffset The offset in the bytestream to read the record from + * @param type The type of VSA extended message record that we are parsing (AA0D, AA0E, AA0F) + * + * @return True if no unhandled failures to parse occurred + */ + bool handleExtendedRecord(uint8_t* const bytes, uint64_t& bytesOffset, VSA::Type type); + + std::vector> vsaRecords; // The vector of records that this parser has parsed + bool hasDeprecatedRecords = false; // Indicates whether records of deprecated types are present in the disk + Settings settings; // The settings used to determine which records to save to records vector + ExtendedMessageState state; // The parse state of all possible extended message sequences + device_eventhandler_t report; // Event handler to report APIEvents +}; + +} // namespace icsneo + +#endif // __cplusplus +#endif // __VSAPARSER_H__ \ No newline at end of file