Skip to content

Commit 90958a1

Browse files
Willmac16Willmac16
committed
Migrate file processing into processFile & Fix/ReEnable most DpCat UTs
1 parent df3ef9a commit 90958a1

File tree

8 files changed

+149
-98
lines changed

8 files changed

+149
-98
lines changed

‎Svc/DpCatalog/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/DpTest*/

‎Svc/DpCatalog/DpCatalog.cpp‎

Lines changed: 115 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,9 @@ DpCatalog ::DpCatalog(const char* const compName)
4242
m_xmitCmdWait(false),
4343
m_xmitBytes(0),
4444
m_xmitOpCode(0),
45-
m_xmitCmdSeq(0) {}
45+
m_xmitCmdSeq(0),
46+
m_pendingFiles(0),
47+
m_pendingDpBytes(0) {}
4648

4749
DpCatalog ::~DpCatalog() {}
4850

@@ -386,20 +388,13 @@ Fw::CmdResponse DpCatalog::fillBinaryTree() {
386388
// keep cumulative number of files
387389
FwSizeType totalFiles = 0;
388390

389-
// file class instance for processing files
390-
Os::File dpFile;
391-
// Working buffer for DP headers
392-
U8 dpBuff[Fw::DpContainer::MIN_PACKET_SIZE]; // Header buffer
393-
Fw::Buffer hdrBuff(dpBuff, sizeof(dpBuff)); // buffer for container header decoding
394-
Fw::DpContainer container; // container object for extracting header fields
395-
396391
// get file listings from file system
397392
for (FwSizeType dir = 0; dir < this->m_numDirectories; dir++) {
398393
// read in each directory and keep track of total
399394
this->log_ACTIVITY_LO_ProcessingDirectory(this->m_directories[dir]);
400395
FwSizeType filesRead = 0;
401-
U32 pendingFiles = 0;
402-
U64 pendingDpBytes = 0;
396+
m_pendingFiles = 0;
397+
m_pendingDpBytes = 0;
403398
U32 filesProcessed = 0;
404399

405400
Os::Directory dpDir;
@@ -434,93 +429,19 @@ Fw::CmdResponse DpCatalog::fillBinaryTree() {
434429
Fw::String fullFile;
435430
fullFile.format("%s/%s", this->m_directories[dir].toChar(), this->m_fileList[file].toChar());
436431

437-
this->log_ACTIVITY_LO_ProcessingFile(fullFile);
438-
439-
// get file size
440-
FwSizeType fileSize = 0;
441-
Os::FileSystem::Status sizeStat = Os::FileSystem::getFileSize(fullFile.toChar(), fileSize);
442-
if (sizeStat != Os::FileSystem::OP_OK) {
443-
this->log_WARNING_HI_FileSizeError(fullFile, sizeStat);
444-
continue;
445-
}
446-
447-
Os::File::Status stat = dpFile.open(fullFile.toChar(), Os::File::OPEN_READ);
448-
if (stat != Os::File::OP_OK) {
449-
this->log_WARNING_HI_FileOpenError(fullFile, stat);
450-
continue;
451-
}
452-
453-
// Read DP header
454-
FwSizeType size = Fw::DpContainer::Header::SIZE;
455-
456-
stat = dpFile.read(dpBuff, size);
457-
if (stat != Os::File::OP_OK) {
458-
this->log_WARNING_HI_FileReadError(fullFile, stat);
459-
dpFile.close();
460-
continue; // maybe next file is fine
461-
}
462-
463-
// if full header isn't read, something's wrong with the file, so skip
464-
if (size != Fw::DpContainer::Header::SIZE) {
465-
this->log_WARNING_HI_FileReadError(fullFile, Os::File::BAD_SIZE);
466-
dpFile.close();
467-
continue; // maybe next file is fine
468-
}
469-
470-
// if all is well, don't need the file any more
471-
dpFile.close();
472-
473-
// give buffer to container instance
474-
container.setBuffer(hdrBuff);
475-
476-
// reset header deserialization in the container
477-
Fw::SerializeStatus desStat = container.deserializeHeader();
478-
if (desStat != Fw::FW_SERIALIZE_OK) {
479-
this->log_WARNING_HI_FileHdrDesError(fullFile, desStat);
480-
}
481-
482-
// add entry to catalog.
483-
DpStateEntry entry;
484-
entry.dir = static_cast<FwIndexType>(dir);
485-
entry.record.set_id(container.getId());
486-
entry.record.set_priority(container.getPriority());
487-
entry.record.set_state(container.getState());
488-
entry.record.set_tSec(container.getTimeTag().getSeconds());
489-
entry.record.set_tSub(container.getTimeTag().getUSeconds());
490-
entry.record.set_size(static_cast<U64>(fileSize));
491-
492-
// check the state file to see if there is transmit state
493-
this->getFileState(entry);
494-
495-
// insert entry into sorted list. if can't insert, quit
496-
bool insertedOk = this->insertEntry(entry);
497-
if (not insertedOk) {
498-
this->log_WARNING_HI_DpInsertError(entry.record);
499-
// clean up and return
500-
this->resetBinaryTree();
501-
this->resetStateFileData();
432+
int ret = processFile(fullFile, dir);
433+
if (ret < 0) {
502434
break;
503435
}
504436

505-
if (entry.record.get_state() == Fw::DpState::UNTRANSMITTED) {
506-
pendingFiles++;
507-
pendingDpBytes += entry.record.get_size();
508-
}
509-
510-
// make sure we haven't exceeded the limit
511-
if (this->m_numDpRecords > this->m_numDpSlots) {
512-
this->log_WARNING_HI_DpCatalogFull(entry.record);
513-
break;
514-
}
515-
516-
filesProcessed++;
437+
filesProcessed += static_cast<U32>(ret);
517438

518439
} // end for each file in a directory
519440

520441
totalFiles += filesProcessed;
521442

522443
this->log_ACTIVITY_HI_ProcessingDirectoryComplete(this->m_directories[dir], static_cast<U32>(totalFiles),
523-
pendingFiles, pendingDpBytes);
444+
m_pendingFiles, m_pendingDpBytes);
524445

525446
// check to see if catalog is full
526447
// that means generated products exceed the catalog size
@@ -534,6 +455,105 @@ Fw::CmdResponse DpCatalog::fillBinaryTree() {
534455

535456
} // end fillBinaryTree()
536457

458+
int DpCatalog::processFile(Fw::String fullFile, FwSizeType dir = DP_MAX_DIRECTORIES) {
459+
// file class instance for processing files
460+
Os::File dpFile;
461+
462+
// Working buffer for DP headers
463+
U8 dpBuff[Fw::DpContainer::MIN_PACKET_SIZE]; // Header buffer
464+
Fw::Buffer hdrBuff(dpBuff, sizeof(dpBuff)); // buffer for container header decoding
465+
Fw::DpContainer container; // container object for extracting header fields
466+
467+
this->log_ACTIVITY_LO_ProcessingFile(fullFile);
468+
469+
// Check if file is in one of our directories
470+
if (dir >= DP_MAX_DIRECTORIES) {
471+
// TODO: Figure out if this DIR is within m_directories
472+
// Currently Skips file
473+
this->log_WARNING_HI_FileOpenError(fullFile, Os::FileSystem::Status::OTHER_ERROR);
474+
return 0;
475+
}
476+
477+
// get file size
478+
FwSizeType fileSize = 0;
479+
Os::FileSystem::Status sizeStat = Os::FileSystem::getFileSize(fullFile.toChar(), fileSize);
480+
if (sizeStat != Os::FileSystem::OP_OK) {
481+
this->log_WARNING_HI_FileSizeError(fullFile, sizeStat);
482+
return 0;
483+
}
484+
485+
Os::File::Status stat = dpFile.open(fullFile.toChar(), Os::File::OPEN_READ);
486+
if (stat != Os::File::OP_OK) {
487+
this->log_WARNING_HI_FileOpenError(fullFile, stat);
488+
return 0;
489+
}
490+
491+
// Read DP header
492+
FwSizeType size = Fw::DpContainer::Header::SIZE;
493+
494+
stat = dpFile.read(dpBuff, size);
495+
if (stat != Os::File::OP_OK) {
496+
this->log_WARNING_HI_FileReadError(fullFile, stat);
497+
dpFile.close();
498+
return 0;
499+
}
500+
501+
// if full header isn't read, something's wrong with the file, so skip
502+
if (size != Fw::DpContainer::Header::SIZE) {
503+
this->log_WARNING_HI_FileReadError(fullFile, Os::File::BAD_SIZE);
504+
dpFile.close();
505+
return 0;
506+
}
507+
508+
// if all is well, don't need the file any more
509+
dpFile.close();
510+
511+
// give buffer to container instance
512+
container.setBuffer(hdrBuff);
513+
514+
// reset header deserialization in the container
515+
Fw::SerializeStatus desStat = container.deserializeHeader();
516+
if (desStat != Fw::FW_SERIALIZE_OK) {
517+
this->log_WARNING_HI_FileHdrDesError(fullFile, desStat);
518+
}
519+
520+
// add entry to catalog.
521+
DpStateEntry entry;
522+
entry.dir = static_cast<FwIndexType>(dir);
523+
entry.record.set_id(container.getId());
524+
entry.record.set_priority(container.getPriority());
525+
entry.record.set_state(container.getState());
526+
entry.record.set_tSec(container.getTimeTag().getSeconds());
527+
entry.record.set_tSub(container.getTimeTag().getUSeconds());
528+
entry.record.set_size(static_cast<U64>(fileSize));
529+
530+
// check the state file to see if there is transmit state
531+
this->getFileState(entry);
532+
533+
// insert entry into sorted list. if can't insert, quit
534+
bool insertedOk = this->insertEntry(entry);
535+
if (not insertedOk) {
536+
this->log_WARNING_HI_DpInsertError(entry.record);
537+
// clean up and return
538+
this->resetBinaryTree();
539+
this->resetStateFileData();
540+
return -1;
541+
}
542+
543+
if (entry.record.get_state() == Fw::DpState::UNTRANSMITTED) {
544+
m_pendingFiles++;
545+
m_pendingDpBytes += entry.record.get_size();
546+
}
547+
548+
// make sure we haven't exceeded the limit
549+
if (this->m_numDpRecords > this->m_numDpSlots) {
550+
this->log_WARNING_HI_DpCatalogFull(entry.record);
551+
return -1;
552+
}
553+
554+
return 1;
555+
}
556+
537557
bool DpCatalog::insertEntry(DpStateEntry& entry) {
538558
// the tree is filled in the following priority order:
539559
// 1. DP priority - lower number is higher priority
@@ -779,6 +799,13 @@ void DpCatalog ::pingIn_handler(FwIndexType portNum, U32 key) {
779799
this->pingOut_out(0, key);
780800
}
781801

802+
void DpCatalog ::addToCat_handler(FwIndexType portNum,
803+
const Fw::StringBase& fileName,
804+
FwDpPriorityType priority,
805+
FwSizeType size) {
806+
// TODO
807+
}
808+
782809
// ----------------------------------------------------------------------
783810
// Handler implementations for commands
784811
// ----------------------------------------------------------------------

‎Svc/DpCatalog/DpCatalog.fpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ module Svc {
4343
@ File Downlink send complete port
4444
async input port fileDone: SendFileComplete
4545

46+
@ DP Writer Add File to Cat
47+
async input port addToCat: DpWritten
48+
4649
# ----------------------------------------------------------------------
4750
# F Prime infrastructure ports
4851
# ----------------------------------------------------------------------

‎Svc/DpCatalog/DpCatalog.hpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,15 @@ class DpCatalog final : public DpCatalogComponentBase {
7070
U32 key //!< Value to return to pinger
7171
) override;
7272

73+
//! Handler implementation for addToCat
74+
//!
75+
//! DP Writer Add File to Cat
76+
void addToCat_handler(FwIndexType portNum, //!< The port number
77+
const Fw::StringBase& fileName, //!< The file name
78+
FwDpPriorityType priority, //!< The priority
79+
FwSizeType size //!< The file size
80+
) override;
81+
7382
private:
7483
// ----------------------------------------------------------------------
7584
// Handler implementations for commands
@@ -133,6 +142,12 @@ class DpCatalog final : public DpCatalogComponentBase {
133142
// Private helpers
134143
// ----------------------------------
135144

145+
/// @brief add entry to sorted list and state file; called on each file in it & upon addToCat
146+
/// @param fullFile full path to file to be processed
147+
/// @param dir directory index in m_directories; DP_MAX_DIRECTORIES if func should search for directory
148+
/// @return -1 for quit, 0 for failure but continue, 1 for success
149+
int processFile(Fw::String fullFile, FwSizeType dir);
150+
136151
/// @brief insert an entry into the sorted list; if it exists, update the metadata
137152
/// @param entry new entry
138153
/// @return failed if couldn't find a slot FIXME: Should we just assert? We should never run out.
@@ -238,6 +253,9 @@ class DpCatalog final : public DpCatalogComponentBase {
238253
U64 m_xmitBytes; //!< bytes transmitted for downlink session
239254
FwOpcodeType m_xmitOpCode; //!< stored xmit command opcode
240255
U32 m_xmitCmdSeq; //!< stored command sequence id
256+
257+
U32 m_pendingFiles;
258+
U64 m_pendingDpBytes;
241259
};
242260

243261
} // namespace Svc

‎Svc/DpCatalog/test/ut/DpCatalogTestMain.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,14 @@ TEST(NominalManual, TreeTestRandom) {
3232

3333
TEST(NominalManual, DISABLED_TreeTestRandomTransmitted) {
3434
Svc::DpCatalogTester tester;
35-
tester.test_NominalManual_DISABLED_TreeTestRandomTransmitted();
35+
tester.test_NominalManual_TreeTestRandomTransmitted();
3636
}
3737

38-
TEST(NominalManual, DISABLED_OneDp) {
38+
TEST(NominalManual, OneDp) {
3939
Svc::DpCatalogTester tester;
4040
Fw::FileNameString dir;
4141
dir = "./DpTest";
42-
Fw::FileNameString stateFile("./DpState");
42+
Fw::FileNameString stateFile("./DpTest/dpState.dat");
4343

4444
Svc::DpCatalogTester::DpSet dpSet;
4545
dpSet.id = 0x123;
@@ -52,12 +52,12 @@ TEST(NominalManual, DISABLED_OneDp) {
5252
tester.readDps(&dir, 1, stateFile, &dpSet, 1);
5353
}
5454

55-
TEST(NominalManual, DISABLED_FiveDp) {
55+
TEST(NominalManual, FiveDp) {
5656
Svc::DpCatalogTester tester;
5757
Fw::FileNameString dirs[2];
5858
dirs[0] = "./DpTest1";
5959
dirs[1] = "./DpTest2";
60-
Fw::FileNameString stateFile("./DpState");
60+
Fw::FileNameString stateFile("./DpTest/dpState.dat");
6161

6262
Svc::DpCatalogTester::DpSet dpSet[5];
6363

‎Svc/DpCatalog/test/ut/DpCatalogTester.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ void DpCatalogTester ::doInit() {
3838
Fw::FileNameString dirs[2];
3939
dirs[0] = "dir0";
4040
dirs[1] = "dir1";
41-
Fw::FileNameString stateFile("dpState.dat");
41+
Fw::FileNameString stateFile("./DpTest/dpState.dat");
4242
this->component.configure(dirs, FW_NUM_ARRAY_ELEMENTS(dirs), stateFile, 100, alloc);
4343
this->component.shutdown();
4444
}
@@ -54,7 +54,7 @@ void DpCatalogTester::testTree(DpCatalog::DpStateEntry* input,
5454

5555
Fw::FileNameString dirs[1];
5656
dirs[0] = "dir0";
57-
Fw::FileNameString stateFile("dpState.dat");
57+
Fw::FileNameString stateFile("./DpTest/dpState.dat");
5858
this->component.configure(dirs, FW_NUM_ARRAY_ELEMENTS(dirs), stateFile, 100, alloc);
5959

6060
// reset tree
@@ -123,7 +123,7 @@ void DpCatalogTester::readDps(Fw::FileNameString* dpDirs,
123123

124124
// dispatch messages
125125
for (FwSizeType msg = 0; msg < numDps; msg++) {
126-
// dispatch file done port call
126+
// dispatch file done port call that is sent on fileOut_handler
127127
this->component.doDispatch();
128128
}
129129

@@ -246,7 +246,7 @@ bool DpCatalogTester ::EntryCompare(const Svc::DpCatalog::DpStateEntry& a, const
246246
}
247247
}
248248

249-
void DpCatalogTester ::test_NominalManual_DISABLED_TreeTestRandomTransmitted() {
249+
void DpCatalogTester ::test_NominalManual_TreeTestRandomTransmitted() {
250250
static const FwIndexType NUM_ENTRIES = 10;
251251
static const FwIndexType NUM_ITERS = 1;
252252

‎Svc/DpCatalog/test/ut/DpCatalogTester.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ class DpCatalogTester : public DpCatalogGTestBase {
128128
// Moved Tests due to private/protected access
129129
// ----------------------------------------------------------------------
130130
static bool EntryCompare(const Svc::DpCatalog::DpStateEntry& a, const Svc::DpCatalog::DpStateEntry& b);
131-
void test_NominalManual_DISABLED_TreeTestRandomTransmitted();
131+
void test_NominalManual_TreeTestRandomTransmitted();
132132
void test_TreeTestManual1();
133133
void test_TreeTestManual2();
134134
void test_TreeTestManual3();

‎Svc/Subtopologies/DataProducts/DataProducts.fpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ module DataProducts{
7171
dpMgr.bufferGetOut[0] -> dpBufferManager.bufferGetCallee
7272
dpMgr.productSendOut[0] -> dpWriter.bufferSendIn
7373
dpWriter.deallocBufferSendOut -> dpBufferManager.bufferSendIn
74+
75+
dpWriter.dpWrittenOut -> dpCat.addToCat
7476
}
7577
} # end topology
7678
} # end DataProducts Subtopology

0 commit comments

Comments
 (0)