#!/usr/bin/env bash

# meant to be run on mercury.dlab.berkeley.edu

DATA_PATH=/var/local/destress/lj-annex/data/events
OUTPUT_PATH=/var/local/destress/combined2/events

MAX_FILE_SIZE=104857600 # 100Mb

mkdir -p $OUTPUT_PATH

# Loop over directories in DATA_PATH
for d in `ls $DATA_PATH`; do
    echo $d
    OUT_FILE_COUNTER=1 # Counter to track when files are split
    for f in `ls $DATA_PATH/$d`; do
        if [ "$OUT_FILE_COUNTER" -eq 1 ]
          then
            OUT_FILE=$OUTPUT_PATH/$d.xml
          else
            OUT_FILE=$OUTPUT_PATH/$d$OUT_FILE_COUNTER.xml
        fi
        cat $DATA_PATH/$d/$f >> $OUT_FILE # Add to combined file

	# Split files if too large
        FILE_SIZE=$(stat -c%s $OUT_FILE)
        if (( "$FILE_SIZE" > "$MAX_FILE_SIZE" ))
          then
           OUT_FILE_COUNTER=$((OUT_FILE_COUNTER+1))
        fi
    done
done

#!/usr/bin/env bash

# meant to be run on mercury.dlab.berkeley.edu

DATA_PATH=/var/local/destress/lj-annex/data/events
OUTPUT_PATH=/home/pierre/combined/events

mkdir -p $OUTPUT_PATH

for f in `ls $DATA_PATH`; do
    echo $f
    # cat $DATA_PATH/$f/* > $OUTPUT_PATH/$f.xml
    find -L $DATA_PATH/$f -maxdepth 1 -type f -print0 | sort -z | xargs -0 cat -- > $OUTPUT_PATH/$f.xml
done


--- 
+++ 
@@ -3,30 +3,13 @@
 # meant to be run on mercury.dlab.berkeley.edu
 
 DATA_PATH=/var/local/destress/lj-annex/data/events
-OUTPUT_PATH=/var/local/destress/combined2/events
-
-MAX_FILE_SIZE=104857600 # 100Mb
+OUTPUT_PATH=/home/pierre/combined/events
 
 mkdir -p $OUTPUT_PATH
 
-# Loop over directories in DATA_PATH
-for d in `ls $DATA_PATH`; do
-    echo $d
-    OUT_FILE_COUNTER=1 # Counter to track when files are split
-    for f in `ls $DATA_PATH/$d`; do
-        if [ "$OUT_FILE_COUNTER" -eq 1 ]
-          then
-            OUT_FILE=$OUTPUT_PATH/$d.xml
-          else
-            OUT_FILE=$OUTPUT_PATH/$d$OUT_FILE_COUNTER.xml
-        fi
-        cat $DATA_PATH/$d/$f >> $OUT_FILE # Add to combined file
+for f in `ls $DATA_PATH`; do
+    echo $f
+    # cat $DATA_PATH/$f/* > $OUTPUT_PATH/$f.xml
+    find -L $DATA_PATH/$f -maxdepth 1 -type f -print0 | sort -z | xargs -0 cat -- > $OUTPUT_PATH/$f.xml
+done
 
-	# Split files if too large
-        FILE_SIZE=$(stat -c%s $OUT_FILE)
-        if (( "$FILE_SIZE" > "$MAX_FILE_SIZE" ))
-          then
-           OUT_FILE_COUNTER=$((OUT_FILE_COUNTER+1))
-        fi
-    done
-done
Concatenate files into chunks no larger than 100Mb.
