evlMain.c

Go to the documentation of this file.

/** \file evlMain.c

\brief Routines for implementing the overall architecture
of Narya firewalling, also some test routines.

*/

#include "evl.h"

/*AutomaticStart*************************************************************/

/*---------------------------------------------------------------------------*/
/* Static function prototypes                                                */
/*---------------------------------------------------------------------------*/

static void preprocess (Pkt_ProcessPkt_t *, Evl_Manager_t *);
static void evlGetPacketsLoop (int maxNumPktsToRead, Evl_Bridge_t * aBridge,
                   array_t * pktArray, u_int32_t slice,
                   int useSynthetic);
static void evlComputeActionsLoop (Evl_BridgeStats_t * stats,
                   Evl_Manager_t * aMgr, Circbuf_t * iBuf,
                   Circbuf_t * ppCache, u_int32_t slice,
                   Circbuf_t * actionBuf);
static void evlPerformActionsLoop (Evl_Manager_t * aMgr,
                   Evl_Bridge_t * aBridge,
                   Circbuf_t * actionBuf, u_int32_t slice);
static void evlQueuingLoop (Evl_BridgeStats_t * stats, Evl_Manager_t * aMgr,
                Circbuf_t * actionBuf, Circbuf_t * ppCache,
                Circbuf_t * oBuf, u_int32_t slice);
static void evlWritePacketsLoop (Evl_Bridge_t * aBridge, u_int32_t slice,
                 int useSynthetic, array_t * ackSeqArray);
static inline void recoverSpace (Pkt_ProcessPkt_t * pp, Circbuf_t * ppCache);
static inline void updateQueuePriorities (Q_Q_t * currentQueue,
                      Heap_t * qHeap);
static inline bool sliceDone (u_int32_t entryTime, u_int32_t slice);
static int  testBuildPrefixAutomaton ();
static int catchSignal (int signo, handler_t handler);
static bool testQueueIsEmpty (Q_Q_t * currentQueue);
static Pkt_ProcessPkt_t *readQueueHead (Q_Q_t * currentQueue, int *classPtr);
static Pkt_ProcessPkt_t *getQueueHead (Q_Q_t * aQ, Q_Flow_t * aFlow,
                       double newRate, int *classPtr);
static void updateRateInfo (util_timing_t * startOfCurrentWindow,
                double timeWindowDuration,
                double *numBytesExitedInWindow, int length);
static int queueInsertPp (Q_Q_t * aQ, Pkt_ProcessPkt_t * pp, int classIndex,
              bool insertAtEnd);
static void pktPrint (char *mesg, Pkt_EthernetHdr_t * pkt);
static void  coreCheck (Evl_BridgeStats_t * stats, Evl_Manager_t * aMgr,
               Circbuf_t * iBuf, Circbuf_t * actionBuf,
               Circbuf_t * oBuf);
static void bridgeSigHandler ();
static int getPacket (char *userPtr , struct pcap_pkthdr *pkthdr,
              u_char * pkt);
static int processPacket (Evl_Bridge_t * aBridge, Pkt_EthernetHdr_t * pkt,
              int length);
static bool doRed (int pktSize, int numBytesInQueue, int maxAllowedBytes);

/*AutomaticEnd***************************************************************/


/**
  * \brief Used by the signal handlers. 

  * It's set by the Evl_Route
  * function, appears to be no other way to tell the signal handler
  * to use this argument.
  */

static Evl_BridgeStats_t *globalStats;


// very useful for debugging, so keep for now 
// static Evl_Manager_t *globalMgr;


/**
  * \brief  Actual firewall code.  Has synthetic input mode for testing.
  */

int
Evl_Route (Evl_Manager_t * aMgr,
       array_t * pktArray,
       int iBufSize,
       int ppCacheSize,
       int oBufSize,
       int actionBufSize,
       int maxNumPktsToRead,
       int numIterations,
       bool useSynthetic,
       array_t * inArray,
       array_t * outArray,
       double basicSlice,
       int getPacketsWeight,
       int computeActionsWeight,
       int performActionsWeight,
       int queuingWeight,
       int writePacketsWeight,
       array_t * ackSeqArray, Evl_BridgeStats_t ** statsPtr)
{
  int i;
  // globalMgr = aMgr;

  Circbuf_t *actionBuf = Circbuf_Init (Pkt_ProcessPkt_t *, actionBufSize);

  Evl_Bridge_t *aBridge = (Evl_Bridge_t *) malloc (sizeof (Evl_Bridge_t));
  aMgr->bridge = aBridge;
  aBridge->iBuf = Circbuf_Init (Pkt_ProcessPkt_t *, iBufSize);
  aBridge->oBuf = Circbuf_Init (Pkt_EthernetHdr_t *, oBufSize);
  aBridge->inArray = !useSynthetic ? Pkt_InitLibPcapArray (inArray)
    : NIL (array_t);
  aBridge->outArray = !useSynthetic ? Pkt_InitLibNetArray (outArray) :
    NIL (array_t);

  aBridge->ppCache = Circbuf_Init (Pkt_ProcessPkt_t *, ppCacheSize);
  aBridge->stats = NIL (Evl_BridgeStats_t);

  Pkt_ProcessPkt_t *ppArray =
    (Pkt_ProcessPkt_t *) malloc (ppCacheSize * sizeof (Pkt_ProcessPkt_t));
  for (i = 0; i < ppCacheSize; i++)
    {
      ppArray[i].applicableRules = array_alloc (int, 0);
      Circbuf_Insert (Pkt_ProcessPkt_t *, aBridge->ppCache, (&(ppArray[i])));
    }

  Evl_BridgeStats_t *stats;
  stats = (Evl_BridgeStats_t *) malloc (sizeof (Evl_BridgeStats_t));
  if (statsPtr != NIL (Evl_BridgeStats_t *))
    {
      *statsPtr = stats;
    }
  globalStats = stats;
  aBridge->stats = stats;
  catchSignal (SIGINT, bridgeSigHandler);

  aMgr->qHeap = Evl_AllocateQueueHeap (aMgr->queueTable);

  u_int32_t slice = util_time_to_ticks (basicSlice);
  u_int32_t gpSlice = slice * getPacketsWeight;
  u_int32_t caSlice = slice * computeActionsWeight;
  u_int32_t paSlice = slice * performActionsWeight;
  u_int32_t qSlice = slice * queuingWeight;

  util_timing_set (&(stats->startTime));
  stats->numEntered = 0;
  stats->numExited = 0;
  stats->bytesWritten = 0;
  stats->numDroppedFullRuleCacheArray = 0;
  stats->numDroppedFullActionBuf = 0;
  stats->numDroppedFullQueue = 0;
  stats->numDroppedFullObuf = 0;
  stats->numDroppedFullIbuf = 0;
  stats->numDroppedBecauseOfAction = 0;
  stats->numDroppedAtWrite = 0;
  stats->numDroppedOverRate = 0;
  stats->numDroppedOverFlowRate = 0;
  (stats->firstPktRead).low = 0;
  (stats->firstPktRead).high = 0;
  (stats->lastPktWritten).low = 0;
  (stats->lastPktWritten).high = 0;
  stats->pcapInfo = aBridge->inArray;

  u_int32_t wpSlice = slice * writePacketsWeight;

  int loopForever = (numIterations == -1);
  while (loopForever || numIterations--)
    {
      evlGetPacketsLoop (maxNumPktsToRead, aBridge, pktArray, gpSlice,
             useSynthetic);
      // if ( stats->numEntered ) util_time( "delta time for call to get" );
      if (stats->numEntered >= maxNumPktsToRead
      && (stats->numEntered == stats->numExited))
    {
      break;
    }
      evlComputeActionsLoop (stats, aMgr, aBridge->iBuf, aBridge->ppCache,
                 caSlice, actionBuf);
      // if ( stats->numEntered ) util_time( "delta time for call to compute" );
      evlPerformActionsLoop (aMgr, aBridge, actionBuf, paSlice);
      // if ( stats->numEntered ) util_time( "delta time for call to actions" );
      evlQueuingLoop (aBridge->stats, aMgr, actionBuf, aBridge->ppCache,
              aBridge->oBuf, qSlice);
      // if ( stats->numEntered ) util_time( "delta time for call to que" );
      evlWritePacketsLoop (aBridge, wpSlice, useSynthetic, ackSeqArray);
      // if ( stats->numEntered ) util_time( "delta time for call to write" );
      if (useSynthetic)
    {
      // coreCheck( stats, aMgr, aBridge->iBuf, actionBuf, aBridge->oBuf );
    }

      // Q_Q_t *aQ;
      // Hash_Lookup( globalMgr->queueTable, (char *) "q0", ( char ** ) & aQ );
      // printf("q0 has %d entries\n", aQ->cosQ->numPktsInQueue );

    }
  util_timing_set (&(stats->finishTime));
  if (useSynthetic)
    {
      // coreCheck( stats, aMgr, aBridge->iBuf, actionBuf, aBridge->oBuf );
    }

  return 0;
  printf ("\n\n|oBuf| = %d\n|iBuf| = %d\n|actionBuf| =  %d\n"
      "Num entered\t= %d\n"
      "Num exited\t= %d\n"
      "Bytes written\t= %d\n"
      "Num Drop: FullRuleCacheArray\t= %d\n"
      "Num Drop: FullActionBuf\t= %d\n"
      "Num Drop: FullIbuf\t= %d\n"
      "Num Drop: FullObuf\t= %d\n"
      "Num Drop: FullQueue\t= %d\n"
      "Num Drop: Action\t= %d\n"
      "Num Drop: OverRate\t= %d\n"
      "Num Drop: OverFlowRate\t= %d\n",
      Circbuf_Num (aBridge->oBuf),
      Circbuf_Num (aBridge->iBuf),
      Circbuf_Num (actionBuf),
      stats->numEntered,
      stats->numExited,
      stats->bytesWritten,
      stats->numDroppedFullRuleCacheArray,
      stats->numDroppedFullActionBuf,
      stats->numDroppedFullIbuf,
      stats->numDroppedFullObuf,
      stats->numDroppedFullQueue,
      stats->numDroppedBecauseOfAction,
      stats->numDroppedOverRate, stats->numDroppedOverFlowRate);
  return 0;

  Q_Q_t *aQ;
  Hash_Lookup (aMgr->queueTable, (char *) "q0", (char **) &aQ);
  printf ("q0 has %d entries\n", aQ->cosQ->numPktsInQueue);
  Pkt_ProcessPkt_t *pp;
  while (1)
    {
      // pp = getQueueHead( aQ, NIL( int ) );
      if (!pp)
    break;
      pktPrint ("", pp->pkt);
    }

  Hash_Lookup (aMgr->queueTable, (char *) "q1", (char **) &aQ);
  printf ("q1 has %d entries\n", aQ->flowQ->numPktsInQueue);
  while (1)
    {
      // pp = getQueueHead( aQ, NIL( int ) );
      if (!pp)
    break;
      pktPrint ("", pp->pkt);
    }

  Hash_Lookup (aMgr->queueTable, (char *) "q2", (char **) &aQ);
  printf ("q2 has %d entries\n", aQ->cosQ->numPktsInQueue);
  while (1)
    {
      // pp = getQueueHead( aQ, NIL( int ) );
      if (!pp)
    break;
      pktPrint ("", pp->pkt);
    }

  return 0;
}


/**
  * \brief  Loop for getting packets off the input buffer
  * 
  */

static void
evlGetPacketsLoop (int maxNumPktsToRead,
           Evl_Bridge_t * aBridge,
           array_t * pktArray, u_int32_t slice, int useSynthetic)
{
  int pktArraySize = (useSynthetic) ? array_n (pktArray) : 0;
  u_int32_t entryTime;
  rdtscl (entryTime);
  int numEnteredStartOfLoop;

  do
    {
      // numEnteredStartOfLoop = globalStats->numEntered;
      numEnteredStartOfLoop = aBridge->stats->numEntered;
      if (aBridge->stats->numEntered >= maxNumPktsToRead)
    {
      break;
    }
      if (Circbuf_IsFull (aBridge->iBuf))
    {
      break;
    }
      if (Circbuf_IsEmpty (aBridge->ppCache))
    {
      break;
    }
      if (useSynthetic)
    {
      Pkt_EthernetHdr_t *aPkt =
        array_fetch (Pkt_EthernetHdr_t *, pktArray,
             (aBridge->stats->numEntered++ % pktArraySize));
      Evl_BridgeInsertPktInIbuf (aBridge, aPkt);

      // Pkt_TcpHdr_t *aTcp = Pkt_EthernetExtractTcp( aPkt );
      // char *payload = Pkt_TcpHdrReadPayload( aTcp );
      // Pkt_IpHdr_t *aIp = Pkt_EthernetExtractIp( aPkt );
      // int dest = aIp->destIp;
      // pktPrint( "Just read in - ", aPkt );

    }
      else
    {
      // get live traffic
      int i;
      static int wastedCalls = 0;
      for (i = 0; i < array_n (aBridge->inArray); i++)
        {
          Pkt_LibPcap_t *aPcapObj =
        array_fetch (Pkt_LibPcap_t *, aBridge->inArray, i);
          aBridge->activeInputInterface = aPcapObj;

          /// \todo - figure out right params for pcap_dispatch
          // void (*dummyFunc)(u_char *, const pcap_pkthdr*, const u_char*);
          // dummyFunc = ( void(*)(u_char *, const pcap_pkthdr*, const u_char*) ) getPacket;

          // void (*dummyFunc)( );
          // void (*dummyFunc)(u_char*, const pcap_pkthdr* , const u_char*);
          // int (*dummyFunc)(char*, pcap_pkthdr*, u_char*);
          
          // dummyFunc = getPacket;
          // pcap_dispatch ( aPcapObj->p, 1, (void(*)(u_char *, const pcap_pkthdr*, const u_char*)) dummyFunc, (u_char *) aBridge );
          pcap_dispatch ( aPcapObj->p, 1, getPacket, (u_char *) aBridge );

        }
      if (numEnteredStartOfLoop == aBridge->stats->numEntered)
        {
          // pcap_dispatch didn't get any more packets, so break out
          wastedCalls++;
          break;
        }
      else
        {
          // printf("%d wasted calls before pickup\n", wastedCalls );
          wastedCalls = 0;
        }
    }
    }
  while (!sliceDone (entryTime, slice));    // do-while loop

}


/**
  * \brief Perform preprocessing
  *
  * Assumptions:
  *   - preprocessing rules appear before regular rules
  *   - no queueing rules in preprocessing
  */

void
preprocess (Pkt_ProcessPkt_t * pp, Evl_Manager_t * mgr)
{
  pp->currentRule = 0;
  Evl_ComputeRuleSetForEth (pp, mgr);

  /// \todo: make sure space is being recovered correctly
  while (pp->currentRule < array_n (pp->applicableRules))
    {
      int ruleId = array_fetch (int, pp->applicableRules, pp->currentRule);
      if (ruleId >= mgr->numPreprocessRules)
    {
      // do not recover space, since this it the preprocess function
      // do reset the applicable rules, otherwise the same rule may
      // appear twice, e,g, on rule
      // tcp 1.2.3.4 any -> any any ()
      // packet with src 1.2.3.4 would appear twice
      array_reset( pp->applicableRules );
      return;
    }
      Evl_Action_t *action =
    array_fetch (Evl_Action_t *, mgr->actionArray, ruleId);
      assert (action->parsedAction->type != Rlp_ActionQueue_c);
      bool dropPkt = Evl_DoAction (action, mgr, mgr->bridge, pp);
      if (dropPkt == true)
    {
      recoverSpace (pp, mgr->bridge->ppCache);
      return;
    }
      pp->currentRule++;
    }
}


/** \brief  Loop for computing actions for packets */

static void
evlComputeActionsLoop (Evl_BridgeStats_t * stats,
               Evl_Manager_t * aMgr,
               Circbuf_t * iBuf,
               Circbuf_t * ppCache,
               u_int32_t slice, Circbuf_t * actionBuf)
{
  u_int32_t entryTime;

  rdtscl (entryTime);
  do
    {

      if (Circbuf_IsEmpty (iBuf))
    {
      break;
    }

      Pkt_ProcessPkt_t *pp = Circbuf_Delete (Pkt_ProcessPkt_t *, iBuf);
      preprocess (pp, aMgr);

      Pkt_EthernetHdr_t *testPkt = pp->pkt;
      int testPktLength = pp->length;

      Evl_ComputeRuleSetForEth (pp, aMgr);

      // assert ( Circbuf_Num( ppCache ) == Circbuf_Num( ruleArrayCache ) );
      pp->pkt = testPkt;
      pp->length = testPktLength;
      pp->currentRule = 0;
      if (!Circbuf_IsFull (actionBuf))
    {
      Circbuf_Insert (Pkt_ProcessPkt_t *, actionBuf, pp);
    }
      else
    {
      // we have no place to put this pkt
      stats->numDroppedFullActionBuf++;
      pktPrint ("Dropped (FullActionBuf) - ", pp->pkt);
      recoverSpace (pp, ppCache);
      break;
    }

    }
  while (!sliceDone (entryTime, slice));
}


/**
  * \brief  Perform the computed actions on packets
  * 
  * Rules must be written in a special order:
  * (any nonqueueing rules in any order)
  * followed by
  * (queueing rules)
  * followed by 
  * (any nonqueueing rule in any order)
  * 
  * When we process rules, we first crank through
  * the nonqueueing rules.  as soon as we get a queuing rule
  * we push the packet in the corresponding queue, and 
  * from then on, it's the queue manager's resp
  * If the push is unsucc, we free the packet.
  * 
  * Later we come back to the remaining rules after the 
  * queueing subsystem is done with then.
  */

static void
evlPerformActionsLoop (Evl_Manager_t * aMgr,
               Evl_Bridge_t * aBridge,
               Circbuf_t * actionBuf, u_int32_t slice)
{
  Evl_BridgeStats_t *stats = aBridge->stats;
  Pkt_ProcessPkt_t *pp;
  Q_Q_t *aQ;
  u_int32_t entryTime;
  rdtscl (entryTime);

  do
    {
      if (Circbuf_IsEmpty (actionBuf))
    {
      return;
    }
      pp = Circbuf_Delete (Pkt_ProcessPkt_t *, actionBuf);
      bool enQueued = false;
      bool dropPkt = false;


      int ruleId;
      Evl_Action_t *action = NIL (Evl_Action_t);
      while ((pp->currentRule < array_n (pp->applicableRules)) && !dropPkt)
    {
      ruleId = array_fetch (int, pp->applicableRules, pp->currentRule);
      action = array_fetch (Evl_Action_t *, aMgr->actionArray, ruleId);
      if (action->parsedAction->type != Rlp_ActionQueue_c)
        {
          dropPkt = Evl_DoAction (action, aMgr, aBridge, pp);
          pp->currentRule++;
        }
      else
        {           //  action->type == Rlp_ActionQueue_c
          if (action->queue == NIL (Q_Q_t))
        {
          Hash_Lookup (aMgr->queueTable,
                 (char *) action->parsedAction->queue,
                 (char **) &action->queue);
        }
          aQ = action->queue;
          // just insert it, let the taildrop take care of limits
          // we dont care about rates, that's for the exiting from the queue
          if (Evl_InsertPacketInQueue (aMgr, aQ, pp, action))
        {
          pp->currentRule++;
        }
          else
        {
          // drop the packet and continue 
          // Pkt_EthernetHdrFree( pp->pkt );
          stats->numDroppedFullQueue++;
          recoverSpace (pp, aBridge->ppCache);
          // this continue was missing led to core dumps in next check
          continue;
        }
          break;        // since now the packet is the responsibility of the queue manager
        }
    }           // while loop

      if (dropPkt)
    {
      stats->numDroppedBecauseOfAction++;
      recoverSpace (pp, aBridge->ppCache);
      /// \todo: cannot right now since it's in the syn traffic recycle array
      // Pkt_EthernetHdrFree( testPkt );
    }
      // tricky - missed the first two checks initially
      else if ((action != NIL (Evl_Action_t))
           && (action->parsedAction->type != Rlp_ActionQueue_c)
           && (!enQueued))
    {
      if (!Circbuf_IsFull (aBridge->oBuf))
        {
          Circbuf_Insert (Pkt_ProcessPkt_t *, aBridge->oBuf, pp);
          // circbuf_insert
          pktPrint ("Put into oBuf - ", pp->pkt);
        }
      else
        {
          stats->numDroppedFullObuf++;
          recoverSpace (pp, aBridge->ppCache);
        }
    }
      // else if -> if it's enqueued, it's the responsibility of the queue manager
    }
  while (!sliceDone (entryTime, slice));
}


/** \brief  Loop for moving packets through queues.  */

static void
evlQueuingLoop (Evl_BridgeStats_t * stats,
        Evl_Manager_t * aMgr,
        Circbuf_t * actionBuf,
        Circbuf_t * ppCache, Circbuf_t * oBuf, u_int32_t slice)
{
  Q_Q_t *currentQueue;
  Q_Flow_t *aFlow = NIL (Q_Flow_t);
  Pkt_ProcessPkt_t *pp;
  double newRate;
  double newFlowRate;
  int classIndex;

  u_int32_t entryTime;

  int ruleId;
  Evl_Action_t *action;

  rdtscl (entryTime);
  do
    {

      if (Heap_Size (aMgr->qHeap) == 0)
    {
      break;
    }

      currentQueue = (Q_Q_t *) Heap_ReadMax (aMgr->qHeap);
      if (testQueueIsEmpty (currentQueue))
    {
      // it's empty so send it back all the way to the bottom of the queue
      currentQueue->currentPriority =
        currentQueue->currentPriority - (Heap_Size (aMgr->qHeap));
      Heap_Heapify (aMgr->qHeap);
      // by continuing, we may waste this whole slice if all queues are empty
      break;
    }
      pp = readQueueHead (currentQueue, &classIndex);

      int pktLength;
      aFlow = NIL (Q_Flow_t);
      if (currentQueue->maxRate != INFINITY)
    {
      // overall queue rate computation
      pktLength = pp->length;
      /// \todo: this assert fails when there's a mixture
      /// of our synthetically created packets and "real packets"
      /// it may be that we're to NTOH incorrectly, and that
      /// the stack has done it for us...
      

      // if it's an IP packet, the captured length should be the set length
      // assert( ! ( Pkt_EthernetReadL3Type( pp->pkt ) == Pkt_L3ProtIp_c ) ||
      //                ( pp->length == Pkt_EthernetPktHdrReadLength( pp->pkt ) ) );

      newRate = Q_ComputeRate (currentQueue->startOfCurrentWindow,
                   currentQueue->timeWindowDuration,
                   currentQueue->numBytesExitedInWindow,
                   pktLength);

      // printf("new Rate = %f\n", newRate );
      if (newRate > currentQueue->maxRate)
        {
          // we're not going to process this queue because it exceeeds the acceptable 
          // incoming rate for this queue
          updateQueuePriorities (currentQueue, aMgr->qHeap);
          break;        // we break so as to avoid getting trapped in a cycle, e.g., for the 1 queue case
        }

      // rate computation for individual flows
      if (currentQueue->type == Q_QueueTypeFlow_c)
        {
          // check if the individual flow's rate will be too high
          aFlow = Q_DrrReadFlow (currentQueue->flowQ, pp->pkt);

          // this check is needed if this packets was the last representative
          // of its flow - in this case, we ignore its rate computation
          if (aFlow != NIL (Q_Flow_t))
        {
          /// \todo - implement fields for flow structure
          // double newFlowRate = Q_ComputeRate( aFlow->startOfCurrentWindow, currentQueue->timeWindowDuration, aFlow->numBytesExitedInDuration, pp->pkt );
          if (newFlowRate > currentQueue->flowMaxRate)
            {
              // we're not going to process this flow because it exceeeds 
              // the acceptable incoming rate for this queue
              updateQueuePriorities (currentQueue, aMgr->qHeap);
              /// \todo - are these breaks the best thing to do?
              break;    // we break so as to avoid getting trapped in a cycle, e.g., for the 1 queue case
            }
        }
        }
    }

      // we're satisfied that pp is acceptable from a rate perspective, 
      // so now we get the queue head, move up the packet rule
      pp = getQueueHead (currentQueue, aFlow, newRate, &classIndex);
      util_timing_set (&stats->lastPktWritten);
      // and update the rate - this is the only place the rate should be updated, as
      // it's a function of the rate at which packets EXIT the queue

      if (pp->currentRule < array_n (pp->applicableRules))
    {
      ruleId = array_fetch (int, pp->applicableRules, pp->currentRule);
      action = array_fetch (Evl_Action_t *, aMgr->actionArray, ruleId);
      if (action->parsedAction->type == Rlp_ActionQueue_c)
        {
          if (action->queue == NIL (Q_Q_t))
        {
          Hash_Lookup (aMgr->queueTable,
                 (char *) action->parsedAction->queue,
                 (char **) &action->queue);
        }
          if (Evl_InsertPacketInQueue (aMgr, action->queue, pp, action))
        {
          pp->currentRule++;
        }
          else
        {
          // drop the packet and continue 
          stats->numDroppedFullQueue++;
          recoverSpace (pp, ppCache);
          // this continue was missing led to core dumps in next check
          continue;
        }
        }
      else
        {
          // it's not queueing rule, so put it in the action buffer
          if (!Circbuf_IsFull (actionBuf))
        {
          Circbuf_Insert (Pkt_ProcessPkt_t *, actionBuf, pp);
        }
          else
        {
          stats->numDroppedFullActionBuf++;
          recoverSpace (pp, ppCache);
        }
        }
    }
      else
    {
      // must have ( pp->currentRule == array_n( pp->applicableRules ) ) 
      // => we've applied all the queue rules to this packet, and there are not any more 
      // queue rules so put pp and applicableRules in the free lists, and move 
      // the packet to the output buffer
      if (!Circbuf_IsFull (oBuf))
        {
          Circbuf_Insert (Pkt_ProcessPkt_t *, oBuf, pp);
        }
      else
        {
          // should sliently discard the packet
          stats->numDroppedFullObuf++;
          recoverSpace (pp, ppCache);
          break;
        }
    }
      updateQueuePriorities (currentQueue, aMgr->qHeap);

    }
  while (!sliceDone (entryTime, slice));
}


/** \brief  Loop for writing out packets */

static void
evlWritePacketsLoop (Evl_Bridge_t * aBridge,
             u_int32_t slice, int useSynthetic, array_t * ackSeqArray)
{
  Evl_BridgeStats_t *stats = aBridge->stats;
  u_int32_t entryTime;
  /// \todo - we're using the first interface as the default output, 
  /// need to document this somewhere
  Pkt_LibNet_t *defaultOutput = NIL (Pkt_LibNet_t);
  if (useSynthetic == false)
    {
      defaultOutput = array_fetch (Pkt_LibNet_t *, aBridge->outArray, 0);
    }

  rdtscl (entryTime);
  do
    {
      if (Circbuf_IsEmpty (aBridge->oBuf))
    {
      break;
    }
      Pkt_ProcessPkt_t *pp =
    Circbuf_Delete (Pkt_ProcessPkt_t *, aBridge->oBuf);
      // need to update stats before we return pp to the cache
      stats->bytesWritten += pp->length;
      if (ackSeqArray)
    {
      array_insert_last (Pkt_EthernetHdr_t *, ackSeqArray, pp->pkt);
    }
      if (useSynthetic == false)
    {
      // we're actually dumping the packet on the line
      /*
         assert( ( Pkt_EthernetReadL3Type( pp->pkt ) != Pkt_L3ProtIp_c ) ||
         ( Pkt_EthernetExtractIp( pp->pkt )->protocol != Pkt_L4ProtTcp_c ) ||
         ( (pp->length == 60 ) && 60 >= Pkt_EthernetPktHdrReadLength( pp->pkt ) ) ||
         ( pp->length == Pkt_EthernetPktHdrReadLength( pp->pkt ) ) );
       */

      Pkt_LibNet_t *outObj = (pp->outIf != NIL (Pkt_LibNet_t)) ?
        pp->outIf : defaultOutput;
      if (0 < Pkt_EthPktWrite (pp->pkt, pp->length, outObj))
        {
          stats->numExited++;
        }
      else
        {
          stats->numDroppedAtWrite++;
        }
      recoverSpace (pp, aBridge->ppCache);
    }

      util_timing_t currentTime;
      util_timing_set (&currentTime);
      double delta =
    util_timing_secs (currentTime) - util_timing_secs (stats->startTime);
      double rate  = 8.0 * stats->bytesWritten / delta;
      // printf("evlWritePacketsLoop: Rate going out is %.1f\n", rate );

      // Pkt_EthernetHdrFree( writePkt );
    }
  while (!sliceDone (entryTime, slice));
}


/**
  * \brief  Add a packet (actually Pkt_ProcessPkt_t *)  to a q - may drop
  * if resources are overused.  
  *
  * Return 1 if succ, 0 if dropped.
  */

int
Evl_InsertPacketInQueue (Evl_Manager_t * aMgr,
             Q_Q_t * aQ,
             Pkt_ProcessPkt_t * pp, Evl_Action_t * action)
{
  int pktSize = Pkt_EthernetPktHdrReadLength (pp->pkt);
  Hash_Lookup (aMgr->queueTable, (char *) action->queue, (char **) &aQ);

  bool checkEntireQueue = false;

  if (aQ->maxAllowedBytes == 0)
    {
      // there's no limit on size
      checkEntireQueue = true;
    }
  else if (aQ->dropMethod == Q_DropTail_c)
    {
      checkEntireQueue =
    ((pktSize + aQ->numBytesInQueue) <= aQ->maxAllowedBytes);
    }
  else
    {
      assert (aQ->dropMethod == Q_DropRed_c);

      /*
       * RED based drop - no drop until half capacity is used.
       * After that, drop prob is $e^{(1/2 - m/M)} / ( e^{1/2} - 1)$, 
       * where m = current use,  M = max allowed
       * This is 0 at 0.5, and tends to 1 as m -> M
       *
       */

      if (2 * (pktSize + aQ->numBytesInQueue) < aQ->maxAllowedBytes)
    {
      // fewer than 0.5 the max byte limit, insert safely
      checkEntireQueue = true;
    }
      checkEntireQueue =
    doRed (pktSize, aQ->numBytesInQueue, aQ->maxAllowedBytes);
    }

  if (!checkEntireQueue)
    {
      return 0;
    }

  if (aQ->type != Q_QueueTypeFlow_c)
    {
      queueInsertPp (aQ, pp, action->parsedAction->classIndex, 1);
      return 1;
    }

  // this is a flow-based queue, and we have not exceeded any of the
  // global bounds on the queue size => focus on individual queue
  bool checkFlowQueue = false;
  Q_Drr_t *aDrr = aQ->flowQ;
  int numFlowsInQueue = Q_DrrNumFlows (aDrr);
  if ((aQ->maxFlows != -1) && (numFlowsInQueue >= aQ->maxFlows))
    {
      checkFlowQueue = false;
    }
  else
    {
      checkFlowQueue = true;
    }

  if (checkFlowQueue && (aQ->maxBytesPerFlow != INFINITY))
    {
      Q_Flow_t *aFlow;
      // when the first packet from the flow is entering, the 
      // lookup fails
      bool succ = Hash_Lookup (aDrr->flowToQueueTable,
                 (char *) pp->pkt,
                 (char **) &aFlow);
      int bytesAlreadyInQueue = succ ? aFlow->numBytes : 0;
      if (aQ->flowDropMethod == Q_DropTail_c)
    {
      if (bytesAlreadyInQueue + pktSize > aQ->maxBytesPerFlow)
        {
          checkFlowQueue = false;
        }
    }
      else
    {
      assert (aQ->flowDropMethod == Q_DropRed_c);
      checkFlowQueue =
        doRed (pktSize, bytesAlreadyInQueue, aQ->maxBytesPerFlow);
    }
    }

  /// \todo: get to buffer limits for individual queues later
  if (checkFlowQueue == true)
    {
      queueInsertPp (aQ, pp, action->parsedAction->classIndex, 1);
      return 1;
    }
  else
    {
      return 0;
    }


  assert (0);           // never get here
}


/**
  * \brief  Recover buffer entries
  * 
  */

static inline void
recoverSpace (Pkt_ProcessPkt_t * pp, Circbuf_t * ppCache)
{
  // printf("Recovering space - pp:%p, pkt:%p\n", pp, pp->pkt );
  array_reset (pp->applicableRules);
  pp->outIf = NIL (Pkt_LibNet_t);

  // Eth_EthPktFree( pp->pkt );
  Circbuf_Insert (Pkt_ProcessPkt_t *, ppCache, pp);
}


/**
  * \brief  Update priorities for queues
  * 
  * The update strategy below has the problem that the lowest priority keeps 
  * decreasing, need to boost all back to near 0 when close in on -MAXINT
  * 
  */

static inline void
updateQueuePriorities (Q_Q_t * currentQueue, Heap_t * qHeap)
{
  currentQueue->currentPriority -= (Heap_Size (qHeap));
  if (!testQueueIsEmpty (currentQueue))
    {
      currentQueue->currentPriority += currentQueue->assignedPriority;
    }
  Heap_Heapify (qHeap);
}


/**
  * \brief  Quit if more time has passed than slice
  * 
  * This is an approximation,
  * as we assume if low-counter has wrapped around we have to stop.
  * 
  * We stop the input work when exitTime has wrapped around,
  * or when exitTime - entryTime is larger than threshold.
  * 
  * The logic for exiting on wraparound is that it'll be fairly rare
  * and will average out over the different operations.
  * 
  */

static inline bool
sliceDone (u_int32_t entryTime, u_int32_t slice)
{
  u_int32_t exitTime;
  rdtscl (exitTime);
  if ((exitTime < entryTime) || ((exitTime - entryTime) > slice))
    {
      return true;
    }
  return false;
}


/**
  * \brief  Routine for testing the manager structure built by 
  * Evl_BuildManager
  * 
  */

int
Evl_TestManager (Evl_Manager_t * aMgr)
{
  Evl_L4Flow_t *t1;

  t1 = (Evl_L4Flow_t *) malloc (sizeof (Evl_L4Flow_t));
  t1->destPort = 80;
  t1->srcPort = 80;
  t1->destIp = Rlp_DotToInt ("192.168.1.1");
  t1->srcIp = 0x00000000;

  int rulesIndex = Evl_ComputeL7RuleSet (aMgr->tcpMgr->destPortRuleTable, t1);
  var_set_t *rules = aMgr->tcpMgr->destPortRuleTable->ruleSets[rulesIndex];

  printf ("Results of lookup:\n");
  int i;
  for (i = 0; i < aMgr->numAllRules; i++)
    {
      char *testRule = array_fetch (char *, aMgr->rawRules, i);
      if (var_set_get_elt (rules, i))
    {
      printf ("Flow %x %x %x %x satisfies %s\n",
          t1->destPort, t1->srcPort, t1->destIp, t1->srcIp, testRule);
    }
    }
  return 0;
}


/**
  * \brief  Test the build prefix automaton routine
  * 
  */

static int 
testBuildPrefixAutomaton ()
{
  util_byte_array_t a;
  util_byte_array_t b;
  util_byte_array_t c;

  a.bytes = "abc";
  b.bytes = "abb";
  c.bytes = "acb";

  a.length = 3;
  b.length = 3;
  c.length = 3;

  array_t *byteArrayArray = array_alloc (util_byte_array_t *, 0);

  array_insert_last (util_byte_array_t *, byteArrayArray, &a);
  array_insert_last (util_byte_array_t *, byteArrayArray, &b);
  array_insert_last (util_byte_array_t *, byteArrayArray, &c);

  Evl_Fsm_t *anFsm = Evl_BuildPrefixAutomaton (byteArrayArray);
  Evl_FsmPrint (anFsm);

  return 0;
}


/**
  * \brief  Catch signals
  * 
  */

static int
catchSignal (int signo, handler_t handler)
{
  struct sigaction action;

  action.sa_handler = ( void (*)(int) ) handler;
  sigemptyset (&action.sa_mask);
  action.sa_flags = 0;

  if (sigaction (signo, &action, NIL (struct sigaction)))
    {
      return -1;
    }
  return 1;
}


/**
  * \brief  Perform action on packet. Return 1 if packet is to be dropped, 
  * 0 otherwise.
  * 
  * Must not be called with queuing actions.
  * 
  */

bool
Evl_DoAction (Evl_Action_t * action,
          Evl_Manager_t * mgr,
          Evl_Bridge_t * bridge, Pkt_ProcessPkt_t * pp)
{
  switch (action->parsedAction->type)
    {
    case Rlp_ActionDrop_c:
      /// \todo: need to figure out how to drop this packet
      /// if we're not doing synthetic runs
      // Pkt_Free( testPkt );
      return 1;
    case Rlp_ActionRoute_c:
      Evl_RoutePkt (action, bridge, pp);
      break;
    case Rlp_ActionUscript_c:
      Evl_UscriptAction (action->parsedAction, mgr, pp);
      break;
    case Rlp_ActionUcode_c:
      // convention is that calling functions returns 1 if pkt is
      // to be dropped 
      return Evl_UcodeAction (action->parsedAction, mgr, pp);
    default:
      printf ("Panic: unknown type of action\n");
      assert (0);
    }               // switch
  return 0;
}


/**
  * \brief check if a queue is empty - works for both types of queues
  */

static bool
testQueueIsEmpty (Q_Q_t * currentQueue)
{
  if (((currentQueue->type == Q_QueueTypeCos_c)
       && Q_CosTestIsEmpty (currentQueue->cosQ))
      ||
      (((currentQueue->type == Q_QueueTypeFlow_c)
    && Q_DrrTestIsEmpty (currentQueue->flowQ))))
    {
      return true;
    }
  else
    {
      return false;
    }
}


/**
  * \brief Read the head of the queue - works for both types of queues. 
  * 
  * Does not change the state of the queue.
  */

static Pkt_ProcessPkt_t *
readQueueHead (Q_Q_t * currentQueue, int *classPtr)
{
  Pkt_ProcessPkt_t *pp = (currentQueue->type == Q_QueueTypeCos_c) ?
    Q_CosReadHead (currentQueue->cosQ,
           classPtr) : Q_DrrReadHead (currentQueue->flowQ);

  return pp;
}


/**
  * \brief Get the head of the queue - works for both types of queues
  */

static Pkt_ProcessPkt_t *
getQueueHead (Q_Q_t * aQ, Q_Flow_t * aFlow, double newRate, int *classPtr)
{
  Pkt_ProcessPkt_t *pp;

  aQ->currentRate = newRate;

  pp = (aQ->type == Q_QueueTypeCos_c) ?
    Q_CosHead (aQ->cosQ, classPtr) : Q_DrrHead (aQ->flowQ);

  if (pp != NIL (Pkt_ProcessPkt_t))
    {
      int length = Pkt_EthernetPktHdrReadLength (pp->pkt);
      aQ->numPktsInQueue--;
      aQ->numBytesInQueue -= length;
      aQ->numBytesExited += length;

      updateRateInfo (&aQ->startOfCurrentWindow,
              aQ->timeWindowDuration,
              &aQ->numBytesExitedInWindow, length);

      // aFlow may be nil if this is the first time this 
      // flow is seen
      if (aFlow != NIL (Q_Flow_t))
    {
      // update the flow's rate info
      updateRateInfo (&aFlow->startOfCurrentWindow,
              aQ->timeWindowDuration,
              &aFlow->numBytesExitedInWindow, length);
    }
    }
  return pp;
}


/**
  * \brief Update rate information
  */

static void
updateRateInfo (util_timing_t * startOfCurrentWindow,
        double timeWindowDuration,
        double *numBytesExitedInWindow, int length)
{
  double startOfCurrentWindowDbl = util_timing_secs (*startOfCurrentWindow);

  // now get current time
  util_timing_t currentTimeUtil;
  util_timing_set (&currentTimeUtil);
  double currentTime = util_timing_secs (currentTimeUtil);

  // see if the difference from when current window started
  // is greater than the window duration time; if so reset the bytes out counter
  // and the window start time.
  if ((currentTime - startOfCurrentWindowDbl) > timeWindowDuration)
    {
      *numBytesExitedInWindow = length;
      *startOfCurrentWindow = currentTimeUtil;
    }
  else
    {
      *numBytesExitedInWindow = length + *numBytesExitedInWindow;
    }
}


/** \brief insert pp into a queue - works for all types of queues */

static int
queueInsertPp (Q_Q_t * aQ, Pkt_ProcessPkt_t * pp, int classIndex, bool insertAtEnd)
{
  int retcode;

  int length = Pkt_EthernetPktHdrReadLength (pp->pkt);

  if (aQ->type == Q_QueueTypeCos_c)
    {
      retcode = Q_CosInsertPpBeginEnd (aQ->cosQ, pp, classIndex, insertAtEnd);
    }
  else if (aQ->type == Q_QueueTypeFlow_c)
    {
      retcode = Q_DrrInsertPpBeginEnd (aQ->flowQ, pp, insertAtEnd);
    }
  else
    {
      assert (0);
    }
  if (retcode)
    {
      aQ->numPktsProcessed++;
      aQ->numPktsInQueue++;
      aQ->numBytesInQueue += length;
      aQ->numBytesEntered += length;
    }
  return retcode;
}


/** \brief Trivial print routine for debugging */

static void
pktPrint (char *mesg, Pkt_EthernetHdr_t * pkt)
{
  return;
  Pkt_IpHdr_t *aIp = Pkt_EthernetExtractIp (pkt);
  int dest = aIp->destIp;
  Pkt_TcpHdr_t *aTcp = Pkt_EthernetExtractTcp (pkt);
  char *payload = Pkt_TcpHdrReadPayload (aTcp);
  printf ("%s pkt %p: payload= %c, destIp = %d\n", mesg, pkt, *payload, dest);
}


/** \brief Core assertion checked */

static void 
coreCheck (Evl_BridgeStats_t * stats,
       Evl_Manager_t * aMgr,
       Circbuf_t * iBuf, Circbuf_t * actionBuf, Circbuf_t * oBuf)
{
  int pktsInQueue = 0;
  char *qName;
  Q_Q_t *aQ;
  st_generator *stgen;

  st_foreach_item (aMgr->queueTable, stgen, (char **) &qName, (char **) &aQ)
  {
    pktsInQueue += aQ->numPktsInQueue;
  }
  // /*
  printf ("\n\n"
      "|oBuf| = %d\n"
      "|iBuf| = %d\n"
      "|actionBuf| = %d\n"
      "Num entered\t= %d\n"
      "Num exited\t= %d\n"
      "Num Drop: FullRuleCacheArray\t= %d\n"
      "Num Drop: FullActionBuf\t= %d\n"
      "Num Drop: FullObuf\t= %d\n"
      "Num Drop: FullQueue\t= %d\n"
      "Num Drop: Action\t= %d\n"
      "|Enqueued Pkts|\t= %d\n",
      Circbuf_Num (oBuf),
      Circbuf_Num (iBuf),
      Circbuf_Num (actionBuf),
      stats->numEntered,
      stats->numExited,
      stats->numDroppedFullRuleCacheArray,
      stats->numDroppedFullActionBuf,
      stats->numDroppedFullObuf,
      stats->numDroppedFullQueue,
      stats->numDroppedBecauseOfAction, pktsInQueue);
  // */
  assert (stats->numDroppedBecauseOfAction +
      pktsInQueue +
      Circbuf_Num (oBuf) +
      Circbuf_Num (iBuf) +
      Circbuf_Num (actionBuf) +
      stats->numExited +
      stats->numDroppedFullRuleCacheArray +
      stats->numDroppedFullActionBuf +
      stats->numDroppedFullQueue +
      stats->numDroppedFullObuf +
      stats->numDroppedOverRate +
      stats->numDroppedOverFlowRate +
      stats->numDroppedFullObuf == stats->numEntered);

  return;

  Hash_Lookup (aMgr->queueTable, (char *) "q0", (char **) &aQ);
  printf ("q0 has %d entries\n", aQ->cosQ->numPktsInQueue);

  Hash_Lookup (aMgr->queueTable, (char *) "q1", (char **) &aQ);
  printf ("q1 has %d entries\n", aQ->flowQ->numPktsInQueue);

  Hash_Lookup (aMgr->queueTable, (char *) "q2", (char **) &aQ);
  printf ("q2 has %d entries\n", aQ->cosQ->numPktsInQueue);

  // code to print the state of the queue
  // changes state of queue!
  /* 
     Pkt_ProcessPkt_t *pp;
     while ( 1 ) {
     pp = getQueueHead( aQ, NIL( int ) );
     if ( !pp ) break;
     pktPrint( "", pp->pkt );
     } 
   */

}


/**
  * \brief  CTRL-C Signal handler for bridge loop
  */

static void
bridgeSigHandler ()
{
  /// \todo: figure out some way to pass an argument to a signal handler
  /// on readinging more, it seems impossible
  static int previousCount = 0;
  printf ("Num packets succ transmitted since last ^C = %d\n",
      globalStats->numExited - previousCount);
  previousCount = globalStats->numExited;;
  printf ("\nBridge stats:\n"
      "Num entered\t= %d\n"
      "Num exited\t= %d\n"
      "Bytes written\t= %d\n"
      "Num Drop: Actual Write\t= %d\n"
      "Num Drop: FullRuleCacheArray\t= %d\n"
      "Num Drop: FullActionBuf\t= %d\n"
      "Num Drop: FullIbuf\t= %d\n"
      "Num Drop: FullObuf\t= %d\n"
      "Num Drop: FullQueue\t= %d\n"
      "Num Drop: Action\t= %d\n"
      "Num Drop: OverRate\t= %d\n"
      "Num Drop: OverFlowRate\t= %d\n",
      globalStats->numEntered,
      globalStats->numExited,
      globalStats->bytesWritten,
      globalStats->numDroppedAtWrite,
      globalStats->numDroppedFullRuleCacheArray,
      globalStats->numDroppedFullActionBuf,
      globalStats->numDroppedFullIbuf,
      globalStats->numDroppedFullObuf,
      globalStats->numDroppedFullQueue,
      globalStats->numDroppedBecauseOfAction,
      globalStats->numDroppedOverRate,
      globalStats->numDroppedOverFlowRate);

  struct pcap_stat pcapStats;
  int i;
  for (i = 0; i < array_n (globalStats->pcapInfo); i++)
    {
      Pkt_LibPcap_t *pcapObj =
    array_fetch (Pkt_LibPcap_t *, globalStats->pcapInfo, i);
      printf ("Stats for %s\n", pcapObj->interfaceName);
      pcap_stats (pcapObj->p, &pcapStats);
      printf ("ps_recv:%d\tps_drop:%d\n", pcapStats.ps_recv,
          pcapStats.ps_drop);
      // packet_ring_stats (pcapObj->p, -1);
    }
  // Q_Q_t *aQ;
  // Hash_Lookup( globalMgr->queueTable, (char *) "q0", ( char ** ) & aQ );
  // printf("q0 has %d entries\n", aQ->cosQ->numPktsInQueue );
  return;
}


/**
  * \brief  Routine called by pcap_dispatch
  */

static int
getPacket (char *userPtr , struct pcap_pkthdr *pkthdr, u_char * pkt)
{
  Pkt_EthernetHdr_t *aPkt = (Pkt_EthernetHdr_t *) pkt;
  Evl_Bridge_t *aBridge = (Evl_Bridge_t *) userPtr;

  int code = processPacket (aBridge, aPkt, pkthdr->len);
  return code;
}


/** \brief  Function that actually processes packets */

static int
processPacket (Evl_Bridge_t * aBridge, Pkt_EthernetHdr_t * pkt, int length)
{
  Circbuf_t *iBuf = aBridge->iBuf;
  Pkt_ProcessPkt_t *pp;

  if (Circbuf_IsFull (iBuf))
    {
      aBridge->stats->numDroppedFullIbuf++;
      return 0;         // input buffer is full
      pktPrint ("Dropped (Full input buffer array) - ", pkt);
    }
  if (Circbuf_IsEmpty (aBridge->ppCache))
    {
      pktPrint ("Dropped (Empty pp cache) - ", pkt);
      return 0;
    }
  pp = Circbuf_Delete (Pkt_ProcessPkt_t *, aBridge->ppCache);
  pp->pkt = pkt;
  pp->length = length;
  pp->inIf = aBridge->activeInputInterface;
  // printf("Got packet, number %d\n", aBridge->stats->numEntered );

  /// \todo when does pcap/socket already do this for us???
  Pkt_NTOH (pkt);
  Circbuf_Insert (Pkt_ProcessPkt_t *, iBuf, pp);
  aBridge->stats->numEntered++;
  return 1;
}


/** \brief Implement Random Early Discard */

static bool
doRed (int pktSize, int numBytesInQueue, int maxAllowedBytes)
{
  bool checkEntireQueue;

  if ((pktSize + numBytesInQueue) > maxAllowedBytes)
    {
      // past the queue size limit
      checkEntireQueue = false;
    }
  else
    {
      // do RED
      double fraction =
    ((double) pktSize + numBytesInQueue) / (double) maxAllowedBytes;
      /// \todo: use lookup tables for this
      double dropProb = (exp (fraction - 0.5) - 1.0) / (exp (0.5) - 1.0);
      double unitRand = util_unit_rand ();
      if (unitRand < dropProb)
    {
      checkEntireQueue = false;
    }
      else
    {
      checkEntireQueue = true;
    }
    }
  return checkEntireQueue;
}


/**
  * \brief Insert a packet into the bridge iBuf, meant
  * to support scripts/code adding control/reset packets.
  */

void
Evl_BridgeInsertPktInIbuf (Evl_Bridge_t * aBridge, Pkt_EthernetHdr_t * aPkt)
{
  if (Circbuf_IsEmpty (aBridge->ppCache))
    {
      printf ("WARNING: Cannot send pkt, pp cache is empty\n");
      return;
    }
  if (Circbuf_IsFull (aBridge->iBuf))
    {
      printf ("WARNING: Cannot send pkt, ibuf is full\n");
      return;
    }

  int length = Pkt_EthernetPktHdrReadLength (aPkt);
  Pkt_ProcessPkt_t *pp =
    Circbuf_Delete (Pkt_ProcessPkt_t *, aBridge->ppCache);
  pp->pkt = aPkt;
  pp->length = length;
  pp->inIf = NIL (Pkt_LibPcap_t);

  Circbuf_Insert (Pkt_ProcessPkt_t *, aBridge->iBuf, pp);
  if (((aBridge->stats->firstPktRead).low == 0)
      && ((aBridge->stats->firstPktRead).high == 0))
    {
      util_timing_set (&aBridge->stats->firstPktRead);
    }
}