LibFlute::File Class Reference

Represents a file being transmitted or received. More...

#include <File.h>

Public Member Functions
	File (LibFlute::FileDeliveryTable::FileEntry entry)
	Create a file from an FDT entry (used for reception) More...
	File (const std::shared_ptr< Transmitter::FileDescription > &file_description)
	Create a file from a Transmitter::FileDescription (used for transmission) More...
	File (uint32_t toi, FecOti fec_oti, std::string content_location, std::string content_type, uint64_t expires, char *data, size_t length, bool copy_data=false)
	Create a file from the given parameters (used for transmission) More...
virtual	~File ()
	Default destructor. More...
void	put_symbol (const EncodingSymbol &symbol)
	Write the data from an encoding symbol into the appropriate place in the buffer. More...
bool	complete () const
	Check if the file is complete. More...
char *	buffer () const
	Get the data buffer. More...
size_t	length () const
	Get the data buffer length. More...
void	encode ()
	Encode the buffer using the Content-Encoding. More...
void	decode ()
	Decode the buffer using the Content-Encoding. More...
bool	is_encoded () const
	Check if the buffer is content encoded. More...
const FecOti &	fec_oti () const
	Get the FEC OTI values. More...
const LibFlute::FileDeliveryTable::FileEntry &	meta () const
	Get the file metadata from its FDT entry. More...
unsigned long	received_at () const
	Timestamp of file reception. More...
void	log_access ()
	Log access to the file by incrementing a counter. More...
unsigned	access_count () const
	Get the access counter value. More...
std::vector< EncodingSymbol >	get_next_symbols (size_t max_size)
	Get the next encoding symbols that fit in max_size bytes. More...
void	mark_completed (const std::vector< EncodingSymbol > &symbols, bool success)
	Mark encoding symbols as completed. More...
void	set_fdt_instance_id (uint16_t id)
	Set the FDT instance ID. More...
uint16_t	fdt_instance_id ()
	Get the FDT instance ID. More...

Detailed Description

Represents a file being transmitted or received.

Definition at line 30 of file File.h.

Constructor & Destructor Documentation

File() [1/3]

LibFlute::File::File

(

LibFlute::FileDeliveryTable::FileEntry

entry

)

Create a file from an FDT entry (used for reception)

Parameters

entry

FDT entry

Definition at line 38 of file File.cpp.

  : _meta( std::move(entry) )
  , _received_at( time(nullptr) )
  , _file_description()
{
  spdlog::debug("Creating File from FileEntry");
  // Allocate a data buffer
  spdlog::debug("Allocating buffer");
  _buffer = (char*)malloc(_meta.fec_oti.transfer_length);
  if (_buffer == nullptr)
  {
    throw std::runtime_error("Failed to allocate file buffer");
  }
  _own_buffer = true;
 
  this->calculate_partitioning();
  this->create_blocks();
}

File() [2/3]

LibFlute::File::File

(

const std::shared_ptr< Transmitter::FileDescription > &

file_description

)

Create a file from a Transmitter::FileDescription (used for transmission)

Parameters

file_description

Transmitter File Description

Definition at line 57 of file File.cpp.

  : _meta()
  , _file_description(file_description)
{
  spdlog::debug("Creating File from FileDescription");
 
  auto length = _file_description->data_length();
  _buffer = (char*)malloc(length);
  if (_buffer == nullptr)
  {
    throw std::runtime_error("No data allocated");
  }
  _own_buffer = true;
  memcpy(_buffer, _file_description->data(), length);
  _meta = _file_description->file_entry();
 
  // for no-code
  if (_meta.fec_oti.encoding_id == FecScheme::CompactNoCode) {
    _meta.fec_oti.transfer_length = length;
  } else {
    throw std::runtime_error("Unsupported FEC scheme");
  }
 
  encode();
 
  calculate_partitioning();
  create_blocks();
}

File() [3/3]

LibFlute::File::File	(	uint32_t	toi,
		FecOti	fec_oti,
		std::string	content_location,
		std::string	content_type,
		uint64_t	expires,
		char *	data,
		size_t	length,
		bool	copy_data = false
	)

Create a file from the given parameters (used for transmission)

Parameters

toi	TOI of the file
content_location	Content location URI to use
content_type	MIME type
expires	Expiry value (in seconds since the NTP epoch)
data	Pointer to the data buffer
length	Length of the buffer
copy_data	Copy the buffer. If false (the default), the caller must ensure the buffer remains valid while the file is being transmitted.

Definition at line 86 of file File.cpp.

  : _own_buffer(false)
  , _meta()
  , _file_description()
{
  spdlog::debug("Creating File from data");
  if (copy_data) {
    spdlog::debug("Allocating buffer");
    _buffer = (char*)malloc(length);
    if (_buffer == nullptr)
    {
      throw std::runtime_error("Failed to allocate file buffer");
    }
    memcpy(_buffer, data, length);
    _own_buffer = true;
  } else {
    _buffer = data;
  }
 
  unsigned char md5[MD5_DIGEST_LENGTH];
  MD5((const unsigned char*)data, length, md5);
 
  _meta.toi = toi;
  _meta.content_location = std::move(content_location);
  _meta.content_type = std::move(content_type);
  _meta.content_length = length;
  _meta.content_md5 = base64_encode(md5, MD5_DIGEST_LENGTH);
  _meta.expires = expires;
  _meta.fec_oti = fec_oti;
 
  // for no-code
  if (_meta.fec_oti.encoding_id == FecScheme::CompactNoCode) { 
    _meta.fec_oti.transfer_length = length;
  } else {
    throw std::runtime_error("Unsupported FEC scheme");
  }
 
  this->calculate_partitioning();
  this->create_blocks();
}

~File()

LibFlute::File::~File ( )

virtual

Default destructor.

Definition at line 134 of file File.cpp.

{
  spdlog::debug("Destroying File");
  if (_own_buffer && _buffer != nullptr)
  {
    spdlog::debug("Freeing buffer");
    free(_buffer);
  }
}

Member Function Documentation

access_count()

unsigned LibFlute::File::access_count ( ) const

inline

Get the access counter value.

Definition at line 132 of file File.h.

{ return _access_count; };

buffer()

char* LibFlute::File::buffer ( ) const

inline

Get the data buffer.

Definition at line 85 of file File.h.

{ return _buffer; };

complete()

bool LibFlute::File::complete ( ) const

inline

Check if the file is complete.

Definition at line 80 of file File.h.

{ return _complete; };

decode()

auto LibFlute::File::decode

(

)

Decode the buffer using the Content-Encoding.

Will check the MD5 sum after decoding, if present.

Definition at line 340 of file File.cpp.

{
  if (!_been_decoded && !_meta.content_encoding.empty()) {
    if (_meta.content_encoding == "gzip" || _meta.content_encoding=="deflate") {
      auto comp_buffer = _buffer;
      bool own_comp = _own_buffer;
      std::shared_ptr<unsigned char[]> decomp_buffer(new unsigned char[16384]);
      z_stream zs = {
    .next_in = reinterpret_cast<unsigned char*>(comp_buffer),
    .avail_in = static_cast<uint32_t>(_meta.fec_oti.transfer_length),
        .next_out = decomp_buffer.get(),
        .avail_out = 16384
      };
      spdlog::debug("Decompressing contents with {}", _meta.content_encoding);
 
      inflateInit2(&zs, 15 | ((_meta.content_encoding == "gzip")?16:0));
      _buffer = nullptr;
      auto zstate = inflate(&zs, Z_FINISH);
      size_t last_out = 0;
      while (zstate == Z_OK) {
        spdlog::debug("Part decompressed: {} bytes", 16384-zs.avail_out);
        _buffer = reinterpret_cast<char*>(realloc(_buffer, zs.total_out));
        memcpy(_buffer+last_out, decomp_buffer.get(), 16384-zs.avail_out);
        last_out = zs.total_out;
        _own_buffer = true;
    zs.avail_out = 16384;
        zs.next_out = decomp_buffer.get();
    zstate = inflate(&zs, Z_FINISH);
      }
      if (zstate==Z_STREAM_END) {
    if (last_out != zs.total_out) {
          spdlog::debug("Finish decompress, last block is {} bytes. Total {} bytes", 16384-zs.avail_out, zs.total_out);
          _buffer = reinterpret_cast<char*>(realloc(_buffer, zs.total_out));
          memcpy(_buffer+last_out, decomp_buffer.get(), 16384-zs.avail_out);
          _own_buffer = true;
        }
        if (!_meta.content_length) {
          _meta.content_length = zs.total_out;
        } else if (_meta.content_length != zs.total_out) {
          spdlog::error("Decompressed length does not match expected Content-Length ({} != {})", _meta.content_length, zs.total_out);
        }
      } else {
        spdlog::error("Error decompressing file {}: {}", _meta.toi, zs.msg);
    throw zs.msg;
      }
 
      if (own_comp) free(comp_buffer);
    } else {
      spdlog::error("Unknown Content-Encoding {}", _meta.content_encoding);
      throw std::runtime_error("Content-Encoding not known");
    }
 
    _been_decoded = true;
    _been_encoded = false;
 
    // Check MD5
    if (!_meta.content_md5.empty()) {
      unsigned char md5[MD5_DIGEST_LENGTH];
      MD5((const unsigned char*)buffer(), length(), md5);
 
      auto content_md5 = base64_decode(_meta.content_md5);
      if (memcmp(md5, content_md5.c_str(), MD5_DIGEST_LENGTH) != 0) {
        spdlog::debug("MD5 mismatch for TOI {}, discarding", _meta.toi);
 
        // MD5 mismatch, try again
        for (auto& block : _source_blocks) {
          for (auto& symbol : block.second.symbols) {
            symbol.second.complete = false;
          }
          block.second.complete = false;
        }
        _complete = false;
      }
    }
  }
}

encode()

auto LibFlute::File::encode

(

)

Encode the buffer using the Content-Encoding.

Definition at line 285 of file File.cpp.

{
  if (!_been_encoded && !_meta.content_encoding.empty()) {
    if (_meta.content_encoding == "gzip" || _meta.content_encoding=="deflate") {
      auto decomp_buffer = _buffer;
      bool own_decomp = _own_buffer;
      std::shared_ptr<unsigned char[]> comp_buffer(new unsigned char[16384]);
      z_stream zs = {
        .next_in = reinterpret_cast<unsigned char*>(decomp_buffer),
        .avail_in = static_cast<uint32_t>(_meta.content_length),
        .next_out = comp_buffer.get(),
        .avail_out = 16384
      };
      spdlog::debug("Compressing contents with {}", _meta.content_encoding);
 
      if (deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY) == Z_OK) {
        _buffer = nullptr;
        auto zstate = deflate(&zs, Z_FINISH);
        size_t last_out = 0;
        while (zstate == Z_OK) {
          spdlog::debug("Part compressed: {} bytes", 16384-zs.avail_out);
          _buffer = reinterpret_cast<char*>(realloc(_buffer, zs.total_out));
          memcpy(_buffer+last_out, comp_buffer.get(), 16384-zs.avail_out);
          last_out = zs.total_out;
          _own_buffer = true;
          zs.avail_out = 16384;
          zs.next_out = comp_buffer.get();
          zstate = deflate(&zs, Z_FINISH);
        }
        if (zstate==Z_STREAM_END) {
          if (last_out != zs.total_out) {
            spdlog::debug("Finish compress, last block is {} bytes. Total {} bytes", 16384-zs.avail_out, zs.total_out);
            _buffer = reinterpret_cast<char*>(realloc(_buffer, zs.total_out));
            memcpy(_buffer+last_out, comp_buffer.get(), 16384-zs.avail_out);
            _own_buffer = true;
          }
          _meta.fec_oti.transfer_length = zs.total_out;
        } else {
          spdlog::error("Error compressing file {}: {}", _meta.toi, zs.msg);
          throw zs.msg;
        }
        deflateEnd(&zs);
 
        if (own_decomp) free(decomp_buffer);
      }
    } else {
      spdlog::error("Unknown Content-Encoding {}", _meta.content_encoding);
      throw std::runtime_error("Content-Encoding not known");
    }
 
    _been_encoded = true;
    _been_decoded = false;
  }
}

fdt_instance_id()

uint16_t LibFlute::File::fdt_instance_id ( )

inline

Get the FDT instance ID.

Definition at line 152 of file File.h.

{ return _fdt_instance_id; };

fec_oti()

const FecOti& LibFlute::File::fec_oti ( ) const

inline

Get the FEC OTI values.

Definition at line 112 of file File.h.

{ return _meta.fec_oti; };

get_next_symbols()

auto LibFlute::File::get_next_symbols

(

size_t

max_size

)

Get the next encoding symbols that fit in max_size bytes.

Definition at line 244 of file File.cpp.

{
  int nof_symbols = std::ceil((float)(max_size - 4) / (float)_meta.fec_oti.encoding_symbol_length);
  auto cnt = 0;
  std::vector<EncodingSymbol> symbols;
  
  for (auto& block : _source_blocks) {
    if (cnt >= nof_symbols) break;
 
    if (!block.second.complete) {
      for (auto& symbol : block.second.symbols) {
        if (cnt >= nof_symbols) break;
    
        if (!symbol.second.complete && !symbol.second.queued) {
          symbols.emplace_back(symbol.first, block.first, symbol.second.data, symbol.second.length, _meta.fec_oti.encoding_id);
          symbol.second.queued = true;
          cnt++;
        }
      }
    }
  }
  return symbols;
 
}

is_encoded()

bool LibFlute::File::is_encoded ( ) const

inline

Check if the buffer is content encoded.

Definition at line 107 of file File.h.

{ return _been_encoded || !_been_decoded; };

length()

size_t LibFlute::File::length ( ) const

inline

Get the data buffer length.

Definition at line 90 of file File.h.

{ return _been_decoded?_meta.content_length:_meta.fec_oti.transfer_length; };

log_access()

void LibFlute::File::log_access ( )

inline

Log access to the file by incrementing a counter.

Definition at line 127 of file File.h.

{ _access_count++; };

mark_completed()

auto LibFlute::File::mark_completed	(	const std::vector< EncodingSymbol > &	symbols,
		bool	success
	)

Mark encoding symbols as completed.

Definition at line 269 of file File.cpp.

{
  for (auto& symbol : symbols) {
    auto block = _source_blocks.find(symbol.source_block_number());
    if (block != _source_blocks.end()) {
      auto sym = block->second.symbols.find(symbol.id());
      if (sym != block->second.symbols.end()) {
        sym->second.queued = false;
        sym->second.complete = success;
      }
      check_source_block_completion(block->second);
      check_file_completion();
    }
  }
}

meta()

const LibFlute::FileDeliveryTable::FileEntry& LibFlute::File::meta ( ) const

inline

Get the file metadata from its FDT entry.

Definition at line 117 of file File.h.

{ return _meta; };

put_symbol()

auto LibFlute::File::put_symbol

(

const EncodingSymbol &

symbol

)

Write the data from an encoding symbol into the appropriate place in the buffer.

Definition at line 144 of file File.cpp.

{
  // Bounds must be ">=", not ">": a valid source-block number is
  // 0.._source_blocks.size()-1, so SBN == size() is already out of range and
  // indexing with it below would be undefined behaviour. Throw a std::exception
  // (not a bare const char*, which escapes the receiver's catch(std::exception&)
  // and terminates the process) so an out-of-range symbol -- e.g. a content
  // object whose on-air symbol layout exceeds its FDT-declared FEC-OTI -- is
  // dropped and logged by the caller instead of crashing the client.
  if (symbol.source_block_number() >= _source_blocks.size()) {
    throw std::runtime_error(fmt::format("FLUTE: source block number {} out of range (have {} blocks)",
                                          symbol.source_block_number(), _source_blocks.size()));
  }
 
  SourceBlock& source_block = _source_blocks[ symbol.source_block_number() ];
 
  if (symbol.id() >= source_block.symbols.size()) {
    throw std::runtime_error(fmt::format("FLUTE: encoding symbol id {} out of range (block {} has {} symbols)",
                                          symbol.id(), symbol.source_block_number(), source_block.symbols.size()));
  }
 
  SourceBlock::Symbol& target_symbol = source_block.symbols[symbol.id()];
 
  if (!target_symbol.complete) {
    symbol.decode_to(target_symbol.data, target_symbol.length);
    target_symbol.complete = true;
 
    check_source_block_completion(source_block);
    check_file_completion();
  }
 
}

received_at()

unsigned long LibFlute::File::received_at ( ) const

inline

Timestamp of file reception.

Definition at line 122 of file File.h.

{ return _received_at; };

set_fdt_instance_id()

void LibFlute::File::set_fdt_instance_id ( uint16_t  id )

inline

Set the FDT instance ID.

Definition at line 147 of file File.h.

{ _fdt_instance_id = id; };

---

The documentation for this class was generated from the following files:

• include/File.h
• src/File.cpp