add occupancy banks and bookkeeping class, add to ClaraWriter and recon-util

common-tools/clara-io/src/main/java/org/jlab/io/clara/Clas12Writer.java

@@ -1,12 +1,10 @@
 package org.jlab.io.clara;
 
-import java.io.File;
 import java.nio.file.Path;
-import java.util.List;
 import org.jlab.clara.std.services.EventWriterException;
 import org.jlab.detector.calib.utils.ConstantsManager;
-import org.jlab.detector.helicity.HelicitySequenceDelayed;
 import org.jlab.detector.serial.SerialHoncho;
+import org.jlab.detector.serial.Occupancer;
 import org.jlab.jnp.hipo4.data.Bank;
 import org.jlab.jnp.hipo4.data.Event;
 import org.jlab.jnp.hipo4.data.SchemaFactory;
@@ -26,6 +24,7 @@
  */
 public class Clas12Writer extends HipoToHipoWriter {
 
+    Occupancer occupancer;
     SerialHoncho serial;
     Bank runConfig;
     ConstantsManager conman;
@@ -37,6 +36,7 @@ public class Clas12Writer extends HipoToHipoWriter {
     int paraTriggerPrescale;
 
     private void init(JSONObject opts) {
+        occupancer = new Occupancer((long)1<<39,100);
         fullSchema = new SchemaFactory();
         fullSchema.initFromDirectory(FileUtils.getEnvironmentPath("CLAS12DIR","etc/bankdefs/hipo4"));
         serial = new SerialHoncho(fullSchema);
@@ -74,6 +74,7 @@ protected HipoWriterSorted createWriter(Path file, JSONObject opts) throws Event
     protected void writeEvent(Object event) throws EventWriterException {
         Event t = serial.read((Event)event);
         if (!t.isEmpty()) writer.addEvent(t, 1);
+        occupancer.process(((Event)event));
         super.writeEvent(event);
         writeRaw((Event)event, t);
     }

common-tools/clas-detector/src/main/java/org/jlab/detector/serial/Occupancer.java

@@ -0,0 +1,192 @@
+package org.jlab.detector.serial;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Map;
+import org.jlab.detector.banks.RawBank;
+import org.jlab.utils.groups.IndexedTable;
+import org.jlab.utils.groups.IndexedTable.IndexedEntry;
+import org.jlab.detector.banks.RawBank.OrderGroups;
+import org.jlab.jnp.hipo4.data.Bank;
+import org.jlab.jnp.hipo4.data.Event;
+import org.jlab.jnp.hipo4.data.Schema;
+import org.jlab.jnp.hipo4.data.SchemaFactory;
+import org.jlab.utils.CLASResources;
+import org.jlab.utils.system.ClasUtilsFile;
+
+/**
+ * Occupancy bookkeeper based on IndexedTable, with I/O helpers for indexed banks.
+ *
+ * @author baltzell
+ */
+public class Occupancer extends ArrayList<Occupancer.OccupanceTable> {
+
+    static final String OCC_BANK_DIR = ClasUtilsFile.getResourceDir("CLAS12DIR","etc/bankdefs/hipo4/singles/occupancy");
+    static final SchemaFactory FULL_SCHEMA = CLASResources.getFullSchema();
+
+    long triggerMask;
+    int prescale;
+    int nevents = 0;
+    Schema cfgSchema;
+
+    /**
+     * 
+     * @param triggerMask
+     * @param prescale 
+     */
+    public Occupancer(long triggerMask, int prescale) {
+        super();
+        this.triggerMask = triggerMask;
+        this.prescale = prescale;
+        init();
+    }
+
+    private void init() {
+        cfgSchema = FULL_SCHEMA.getSchema("RUN::config");
+        try {
+            // Initialize list, by reading occupancy bank filenames from a directory
+            // and stripping ".json" suffix and "OCC::" prefix to get hit bank name:
+            addAll(Files.list(Paths.get(OCC_BANK_DIR))
+                    .filter(Files::isRegularFile)
+                    .map(p -> p.getFileName().toString())
+                    .map(s -> s.substring(0, s.length()-5))
+                    .map(s -> s.substring(5, s.length()))
+                    .map(OccupanceTable::new).toList());
+        } catch (IOException ex) {
+            System.getLogger(Occupancer.class.getName()).log(System.Logger.Level.ERROR, (String) null, ex);
+        }
+    }
+
+    public boolean process(Event event) {
+        Bank cfgBank = event.read(new Bank(cfgSchema, 1));
+        boolean passTrigger = cfgBank.getRows() > 0 && 0 != (triggerMask & cfgBank.getLong("trigger",0));
+        if (triggerMask <= 0 || passTrigger) {
+            int size = size();
+            for (int i=0; i<size; i++) get(i).fill(event);
+            if (++nevents % prescale == 0) {
+                for (int i=0; i<size; i++) {
+                    if (get(i).getTable().getRowCount() > 0) {
+                        event.write(get(i).create(nevents));
+                    }
+                    get(i).reset();
+                }
+                nevents = 0;
+            }
+        }
+        return true;
+    }
+
+    public void reset() {
+        int size = size();
+        for (int i=0; i<size; i++) {
+            get(i).reset();
+            nevents = 0;
+        }
+    }
+
+    public static class OccupanceTable {
+
+        Schema occSchema;
+        Schema hitSchema;
+        IndexedTable table;
+
+        /**
+         * A 3- or 4-index table, depending on "order".
+         * @param hitBank name of the hit bank
+         */
+        public OccupanceTable(String hitBank) {
+            hitSchema = FULL_SCHEMA.getSchema(hitBank);
+            occSchema = FULL_SCHEMA.getSchema("OCC::" + hitBank);
+            table = new IndexedTable(occSchema.hasEntry("order")?4:3, new String[]{"occ/F"});
+        }
+
+        /**
+         * An N-index table.
+         * @param hitBank name of the hit bank
+         * @param indexCount number of inidices in the hit bank
+         */
+        public OccupanceTable(String hitBank, int indexCount) {
+            hitSchema = FULL_SCHEMA.getSchema(hitBank);
+            occSchema = FULL_SCHEMA.getSchema("OCC::" + hitBank);
+            table = new IndexedTable(indexCount, new String[]{"occ/F"});
+        }
+
+        public final IndexedTable getTable() { return table; }
+
+        /**
+         * Zero the occupancy table.
+         */
+        public final void reset() {
+            table = new IndexedTable(table.getList().getIndexSize(), new String[]{"occ/F"});
+        }
+
+        /**
+         * Get the occupancy table, normalized by number of events.
+         * @param events
+         * @return
+         */
+        public final IndexedTable getOccupancy(long events) {
+            IndexedTable t = new IndexedTable(table.getList().getIndexSize(), new String[]{"occ/F"});
+            for (long hash : ((Map<Long,IndexedEntry>)table.getList().getMap()).keySet()) {
+                t.addEntry(IndexedTable.DEFAULT_GENERATOR.getIndices(hash, table.getList().getIndexSize()));
+                t.setDoubleValueByHash((table.getDoubleValueByHash(0, hash))/events, 0, hash);
+            }
+            return t;
+        }
+
+        /**
+         * Fill the occupancy table.
+         * @param index
+         */
+        public final void fill(int... index) {
+            for (int i=0; i<index.length; i++) if (index[i] < 0) return;
+            final long hash = IndexedTable.DEFAULT_GENERATOR.hashCode(index);
+            if (!table.hasEntryByHash(hash)) {
+                table.addEntry(index);
+                table.setDoubleValueByHash(0.0d, 0, hash);
+            }
+            table.setDoubleValueByHash(table.getDoubleValueByHash(0, hash) + 1.0, 0, hash);
+        }
+
+        /**
+         * Fill occupancy table from a user-defined bank.
+         * @param event
+         */
+        public void fill(Event event) {
+            RawBank bank = new RawBank(hitSchema, 1000, OrderGroups.DEFAULT);
+            bank.read(event);
+            final int rows = bank.rows();
+            int[] idx = new int[table.getList().getIndexSize()];
+            for (int i=0; i<rows; i++) {
+                for (int j=0; j<table.getList().getIndexSize(); j++) {
+                    if (j==2) idx[j] = bank.getShort(j,i);
+                    else idx[j] = bank.getByte(j,i);
+                }
+                fill(idx);
+            }
+        }
+
+        /**
+         * Get an occupancy bank, normalized by number of events.
+         * @param events
+         * @return
+         */
+        public Bank create(long events) {
+            Bank b = new Bank(occSchema, table.getRowCount());
+            Map<Long,IndexedEntry> m = table.getList().getMap();
+            int i = 0;
+            for (long hash : m.keySet()) {
+                int[] idx = IndexedTable.DEFAULT_GENERATOR.getIndices(hash, table.getList().getIndexSize());
+                for (int j=0; j<table.getList().getIndexSize(); j++) {
+                    if (j == 2) b.putShort(j, i, (short)idx[j]);
+                    else b.putByte(j, i, (byte)idx[j]);
+                }
+                b.putFloat(table.getList().getIndexSize(), i, ((float)m.get(hash).getValue(0).intValue())/events);
+                i++;
+            }
+            return b;
+        }
+    }
+}

common-tools/clas-reco/src/main/java/org/jlab/clas/reco/EngineProcessor.java

@@ -19,6 +19,7 @@
 import java.util.Arrays;
 import org.jlab.coda.jevio.EvioException;
 import org.jlab.detector.decode.CLASDecoder4;
+import org.jlab.detector.serial.Occupancer;
 import org.jlab.io.evio.EvioDataEvent;
 import org.jlab.io.evio.EvioSource;
 import org.jlab.io.hipo.HipoDataEvent;
@@ -42,7 +43,8 @@ public class EngineProcessor {
     private SchemaFactory banksToKeep = null;
     private final List<String> schemaExempt = Arrays.asList("RUN::config","DC::tdc");
 
-    private CLASDecoder4 decoder = new CLASDecoder4();
+    private final CLASDecoder4 decoder = new CLASDecoder4();
+    private final Occupancer occupancer = new Occupancer((long)1<<39,100);
 
     public EngineProcessor(){}
 
@@ -316,6 +318,7 @@ public void processFile(String file, String output){
     public void processEvent(DataEvent event, HipoDataSync writer) {
         processEvent(event);
         removeBanks(event);
+        occupancer.process(((HipoDataEvent)event).getHipoEvent());
         writer.writeEvent(event);
     }