Example Documentation

Suppose we want to create a series of frames to store the content of (say) two data channels, named ch1, ch2.

Assume that data for channel ch1 are contained in one single file ch1.dat, as a sequence of number in ASCII format, separated by spaces and/or tabs and/or newlines. Let these data be sampled at 2048 Hz.

For the other channel, data are contained in several files named ch2_a.dat, ch2_b.dat, ch2_c.dat, and assume they are sampled at 4096 Hz.

 ascii2frame -channel ch1 -sampling 2048 -i ch1.dat \
             -channel ch2 -sampling 4096 -i ch2_a.dat -i ch2_b.dat -i ch2_c.dat \
             -frames_per_file 8 -seconds_per_frame 2

Depending on the length of the files ch1.dat, ch2_a.dat ..., several frame files shall be created: typically with names like

 P-677135598-8-2.F  P-677135600-8-2.F  P-677135602-8-2.F ...

/******************************************************************************
 *
 * Copyright (C) 2001 by Andrea Vicere'.
 *
 * Permission to use, copy, modify, and distribute this software and its
 * documentation under the terms of the GNU General Public License is hereby 
 * granted. No representations are made about the suitability of this software 
 * for any purpose. It is provided "as is" without express or implied warranty.
 * See the GNU General Public License for more details.
 *
 */
#include <stdlib.h>
#include <string.h>
#include "fbe.h"
#include "ascii_read.h"

#define MAX_FILES 128
#define MAX_CHANNEL 32


void Help(FILE *stream, char *name)
{
  fprintf(stream,"\n"
          "This program converts ASCII files containing data for different\n"
          "channels into frames.\n"
          "Each channel can be read from one or more ASCII files.\n"
          "Options are available to set separately sampling, gain,\n"
          "type of storage for each channel.\n"
          "The output is in one or more files, each containing one or more\n"
          "frames, at the discretion of the user\n"
          "-------------------------------------------------------------------- \n"
          "Usage: %s -detector <name> : set the originator of data\n"
          "          -channel <name> : create a channel with this name\n"
          "          -sampling <float> : set the sampling for this channel\n"
          "          -gain <float> : set the gain for the current channel\n"
          "          -bits <int> : bits to use to encode this channel\n"
          "          -type <adc|sim|proc> : set the type of channel\n"
          "          -i <filename> : add file to the inputs for this channel\n"
          "          -skip <int> : skip these many chars at the beginning\n"
          "             of the current file.\n"
          "          -limit <int> : limit on the number of items to read\n"
          "             from the current file (e.g.: -limit 5 means read 5 numbers.)\n"
          "          -o <filename> : output base filename>\n"
          "          -compress <int> : compression type\n"
          "          -gzip <int> : gzip compression level>\n"
          "          -debug <int> : debug level [default 0]\n"
          "          -seconds_per_frame <int> : seconds for each frame\n"
          "          -frames_per_file <int> : how many frames per file\n"
          "          -GTimeS <int> : starting frame time in GPS seconds \n"
          "          -GTimeN <int> : starting frame time, GPS nanoseconds\n"
          "-------------------------------------------------------------------- \n"
          "Remarks:\n"
          "         -compress Compression types are 0 (no compression), 1 (gzip)\n"
          "            3 (differentiation+gzip), 5 (gzip for float+zero suppress for int)\n"
          "            6 (zero suppress for int). The default is 3\n"
          "         -gzip The gzip level ranges from 0 (no compression) to 9 (maximum, slow).\n"
          "            Default is 1\n"
          "         -type Can be adc, proc, sim. Default [sim]\n"
          "               Use use adc  for raw data (adc counts)\n"
          "                       proc for processed data\n"
          "                       sim  for simulated data.\n"  
          "         -bits Allowed values depend on the type: if type is\n"
          "               proc or sim: if bits < -32, double format is used,\n"
          "                        otherwise float.\n"
          "               adc: if bits > 16, long int are used\n"
          "                    if bits > 8, short int\n"
          "                    if bits > 0, char\n"
          "                    if bits < -32, double\n"
          "                    default float\n",
          name);
  return;
}

struct Parameters {
  char * detector;
  int channelCounter;
  struct asciiFile  ** asciiInput;
  char * outputFileBaseName;
  int compress;
  int gzip_level;
  int debug;
  double seconds_per_frame;
  int frames_per_file;
  int GTimeS;
  int GTimeN;
  double * sampling;
  char ** type;
  double * gain;
  int * bits;
};

struct Parameters * ParametersNew(int argc, char *argv[])
{
  struct Parameters * this;
  int i, f, c;
  int fileCounter = 0;
  long * limit;
  long * skip;
  char * channelName = NULL;
  char ** fileName = NULL;
  this = (struct Parameters*) malloc(sizeof(struct Parameters));
  
  /* default values */
  this->channelCounter = 0;
  this->outputFileBaseName = strdup("P");
  this->detector = strdup("unknown");
  this->compress = 3;
  this->gzip_level = 1;
  this->debug = 0;
  this->seconds_per_frame = 1.0;
  this->frames_per_file = 1;
  this->GTimeS = -1; /* means read it from the system clock */
  this->GTimeN = -1; /* means set to zero */
  this->asciiInput = (struct asciiFile**) malloc(sizeof(struct asciiFile*)*MAX_CHANNEL);
  this->sampling = (double*) malloc(sizeof(double)*MAX_CHANNEL);
  this->gain = (double*) malloc(sizeof(double)*MAX_CHANNEL);
  this->bits = (int*) malloc(sizeof(int)*MAX_CHANNEL);
  this->type = (char**) malloc(sizeof(char*)*MAX_CHANNEL);
  limit = (long*) malloc(sizeof(long)*MAX_CHANNEL);
  fileName = (char**) malloc(sizeof(char*)*MAX_CHANNEL);
  for(c = 0; c < MAX_CHANNEL; c++)
    {
      this->asciiInput[c] = NULL;
      this->sampling[c] = 2048.0;
      this->gain[c] = 1.0;
      this->bits[c] = -32;
      this->type[c] = NULL; /* means should be allocated to default */
      limit[c] = 2147483647L;  /* 2^31, means read everything */
      fileName[c] = NULL;
    }
  skip = (long*) malloc(sizeof(int)*MAX_FILES);
  for(f = 0; f < MAX_FILES; f++)
    skip[f] = 0;

  /* issue a message of warning if there are no arguments */
  if(argc == 1)
    {
      fprintf(stderr,"%s:usage: issue \"%s -h | more\" for help\n",
              argv[0], argv[0]);
      exit(0);
    }

  /* parsing of the command line */
  for(i=1; i<argc; i++)
    {
      if(strcmp(argv[i],"-h")==0)
        {
          Help(stdout, argv[0]);
          exit(0);
        }
      else if(strcmp(argv[i],"-detector")==0)
        this->detector=strdup(argv[++i]);
      else if(strcmp(argv[i],"-frames_per_file")==0)
        this->frames_per_file=atoi(argv[++i]);
      else if(strcmp(argv[i],"-GTimeS")==0)
        this->GTimeS = atol(argv[++i]);
      else if(strcmp(argv[i],"-GTimeN")==0)
        this->GTimeN = atol(argv[++i]);
      else if(strcmp(argv[i],"-seconds_per_frame")==0)
        this->seconds_per_frame=atof(argv[++i]);
      else if(strcmp(argv[i],"-skip")==0)
        {
          if(fileCounter == 0)
            {
              fprintf(stderr,
                      "%s: please give -i first\n"
                      "  issue \"%s -h | more\" for help\n", 
                      argv[0], argv[0]);
              exit(0);
            }
          skip[fileCounter - 1] = atol(argv[++i]);
        }
      else if(strcmp(argv[i],"-channel")==0)
        {
          /* open a new stream: files defined later are
             assumed to belong to this stream */

          /* Was there a stream open? */
          if(channelName != NULL)
            {
              /* close this channel: create the appropriate structure */
              if((this->asciiInput[this->channelCounter] = 
                  asciiFileNew(channelName, fileCounter)) == NULL)
                exit(1);
              for(f=0; f< fileCounter; f++)
                asciiFileIni(this->asciiInput[this->channelCounter],
                             fileName[f], limit[f], skip[f]);
              this->channelCounter++;
              free(channelName);
              channelName=NULL;
            }
          channelName = strdup(argv[++i]);
          /* reset counters */
          fileCounter = 0;
        }
      else if(strcmp(argv[i],"-i")==0)
        {
          if(channelName == NULL)
            {
              fprintf(stderr,
                      "%s: before adding files please define the channel\n"
                      "    issue \"%s -h | more\" for help",
                      argv[0], argv[0]);
              exit(1);
            }
          fileCounter++;
          fileName[fileCounter-1] = strdup(argv[++i]);
          if(fileCounter == MAX_FILES)
            {
              fprintf(stderr,
                      "%s: too many input files. Increase MAX_FILES please.\n", 
                      argv[0]);
              exit(1);
            }
        }
      else if(strcmp(argv[i],"-limit")==0)
        {
          if(fileCounter == 0)
            {
              fprintf(stderr,
                      "%s: before setting limits please give -i\n"
                      "    issue \"%s -h | more\" for help",
                      argv[0], argv[0]);
              exit(1);
            }
          limit[fileCounter-1] = atol(argv[++i]);
        }
      else if(strcmp(argv[i],"-compress")==0)
        this->compress = atoi(argv[++i]);
      else if(strcmp(argv[i],"-debug")==0)
        this->debug = atoi(argv[++i]);
      else if(strcmp(argv[i],"-gzip_level")==0)
        this->gzip_level = atoi(argv[++i]);
      else if(strcmp(argv[i],"-sampling")==0)
        {
          if(channelName == NULL)
            {
              fprintf(stderr,
                      "%s: before setting the sampling you should define a channel\n"
                      "   issue \"%s -h | more\" for help\n",
                      argv[0],argv[0]);
              exit(1);
            }
          this->sampling[this->channelCounter] = atof(argv[++i]);
        }
      else if(strcmp(argv[i],"-gain")==0)
        {
          if(channelName == NULL)
            {
              fprintf(stderr,
                      "%s: before setting the gain you should define a channel.\n"
                      "   issue \"%s -h | more \" for help", 
                      argv[0], argv[0]);
              exit(1);
            }
          this->gain[this->channelCounter] = atof(argv[++i]);
        }
      else if(strcmp(argv[i],"-type")==0)
        {
          if(channelName == NULL)
            {
              fprintf(stderr,
                      "%s: before setting the type you should define a channel.\n"
                      "   issue \"%s -h | more \" for help", 
                      argv[0], argv[0]);
              exit(1);
            }
          this->type[this->channelCounter] = strdup(argv[++i]);
        }
      else if(strcmp(argv[i],"-bits")==0)
        {
          if(channelName == NULL)
            {
              fprintf(stderr,
                      "%s: before setting the bits you should define a channel.\n"
                      "   issue \"%s -h | more \" for help", 
                      argv[0], argv[0]);
              exit(1);
            }
          this->bits[this->channelCounter] = atoi(argv[++i]);
        }
      else
        {
          fprintf(stderr,"%s: unrecognized option %s\n"
                  "   issue \"%s -h | more\" for help\n",
                  argv[0], argv[i], argv[0]);
          exit(1);
        }
    }

  /* is there a channel open? */
  if(channelName != NULL)
    {
      /* close this channel: create the appropriate structure */
      if((this->asciiInput[this->channelCounter] = 
          asciiFileNew(channelName, fileCounter)) == NULL)
        exit(1);
      for(f=0; f < fileCounter; f++)
        asciiFileIni(this->asciiInput[this->channelCounter], 
                     fileName[f], limit[f], skip[f]);
      this->channelCounter++;
      free(channelName);
    }

  /* set some defaults which can be only now defined */
  for(c = 0; c < this->channelCounter; c++)
    if(this->type[c] == NULL)
      this->type[c] = strdup("sim");

  /* free unnecessary memory */
  free(limit);
  free(skip);
  free(fileName);

  /* well, don't forget! */
  return this;
}

/* simple program to convert ASCII files into frames */
int main(int argc, char * argv[])
{
  /* declarations */
  int errorFlag = 0;
  int filledFlag;
  int errorFlagChannel[MAX_CHANNEL];
  char buffer[128];
  int i, c;
  long nDataPerFrame[MAX_CHANNEL];
  struct Parameters * param = NULL;
  struct FbeBase * fbeBuilder;
  struct FbeBase * sampler[MAX_CHANNEL];
  struct FbeBase * oFile;

  /* allocation and initialization of the parameters */
  param = (struct Parameters *) ParametersNew(argc, argv);

  /* basic initializations */
  for(c = 0; c < MAX_CHANNEL; c++)
    errorFlagChannel[c] = 0;

  /* create and initialize the frame builder  */
  if((fbeBuilder = FbeBuilderNew(param->seconds_per_frame, NULL))==NULL)
    exit(1);
  sprintf(buffer,
          "debug=%d detector=%s GTimeS=%d GTimeN=%d run=0 frame=0",
          param->debug, param->detector, param->GTimeS, param->GTimeN);
  errorFlag |= FbeBuilderSet(fbeBuilder, buffer, NULL);
  if(errorFlag & FBE_FATAL)
    exit(errorFlag);

  /* create and initialize the samplers */
  for(c=0; c <param->channelCounter; c++)
    if((sampler[c] = FbeSamplerNew(1.0/param->sampling[c], 
                                   fbeBuilder->frameH))==NULL)
      exit(1);
  for(c=0; c < param->channelCounter; c++)
    {
      sprintf(buffer, "debug=%d name=%s gain=%g bits=%d type=%s", 
              param->debug, param->asciiInput[c]->channelName, param->gain[c], 
              param->bits[c], param->type[c]);
      errorFlagChannel[c] |= 
        FbeSamplerSet(sampler[c], buffer, 
                      (int (*)(void*,double*)) asciiFileRead);
      if(errorFlagChannel[c] & FBE_FATAL)
        exit(errorFlagChannel[c]);
    }

  /* create and initialize the file writer */
  if((oFile =
      FbeFilerNew(param->seconds_per_frame*
                  param->frames_per_file, fbeBuilder->frameH))==NULL)
    exit(1);
  sprintf(buffer,
          "debug=%d name=dummy fileName=%s compress=%d gzip_level=%d frames_per_file=%d",
          param->debug, param->outputFileBaseName, param->compress, 
          param->gzip_level, param->frames_per_file);
  errorFlag |= FbeFilerSet(oFile, buffer, NULL);
  if(errorFlag & FBE_FATAL)
    exit(errorFlag);

  /* any errors till now ? Report and reset */
  if(errorFlag)
    {
      fprintf(stderr,
              "%s: there were non fatal initialization errors: continuing\n",
              argv[0]);
      errorFlag = 0;
    }
  for(c=0; c<param->channelCounter; c++)
    if(errorFlagChannel[c])
      {
        fprintf(stderr,
                "%s: channel %d had a non fatal error: continuing\n",
                argv[0], c);
        errorFlagChannel[c] = 0;
      }

  /* how many data of this channel in a frame? */
  for(c=0; c<param->channelCounter; c++)
    nDataPerFrame[c] = 
      (long) (param->seconds_per_frame * param->sampling[c]);

  /* cycle reading and filling the frames */
  while(!errorFlag)
    {
      /* start a new frame */
      errorFlag |= FbeBuilderSim(fbeBuilder, NULL);
      if(errorFlag & FBE_FATAL)
        exit(errorFlag);

      /* cycle over the channels and fill them */
      filledFlag = 0;
      for(c = 0; c < param->channelCounter; c++)
        if(!errorFlagChannel[c]) /* skip if there are errors */
          for(i = 0; i < nDataPerFrame[c]; i++)
            {
              /* read one number in the channel, and stop
                 on errors */
              if(errorFlagChannel[c] |= 
                 FbeSamplerSim(sampler[c], param->asciiInput[c]))
                break;
              else
                filledFlag = 1;
            }

      /* if the frame has been filled, even partially,
         we close it and write on the file */
      if(filledFlag)
        errorFlag |= FbeFilerSim(oFile, NULL);

      /* but fatal errors can't be tolerated */
      if(errorFlag & FBE_FATAL)
        exit(errorFlag);

      /* are there warnings till now? If yes, we have finished */
      if(errorFlag)
        break;

      /* if any channel reports no error, it means we have
         still data to read */
      errorFlag = 1;
      for(c = 0; c < param->channelCounter; c++)
        errorFlag *= errorFlagChannel[c];
    }

  /* close the file */
  FbeFilerFree(oFile);

  /* free Fbe memory, just to check that there are no memory leaks */
  FbeBuilderFree(fbeBuilder);
  for(c=0; c<param->channelCounter; c++)
    FbeSamplerFree(sampler[c]);

  /* free the asciiFile(s) */
  for(c=0; c<param->channelCounter; c++)
    asciiFileFree(param->asciiInput[c]);

  exit(0);
}

ascii2frame.c

The ascii2frame tool