Differences

This shows you the differences between two versions of the page.

--- code:work:lvisf:2024:field_processing:arcsix:start [2024/06/14 18:28] – [ARCSIX Field Processing] david
+++ code:work:lvisf:2024:field_processing:arcsix:start [2024/10/15 17:30] (current) – [Final Raid Inventory] david
@@ Line 1: / Line 1: @@
 ====== ARCSIX Field Processing ======
+  * [[code:work:lvisf:2024:field_processing:arcsix2:summary:start|ARCSIX-2 Data Summary]]
+  * [[code:work:lvisf:2024:field_processing:arcsix2:summary:alignment|ARCSIX-2 Alignment Summary]]
   * Backup Drives "/media/lvis/A6_BACKUP01"
   * Backup Drives "/media/lvis/A6_BACKUP02"
@@ Line 110: / Line 112: @@
 rsync -vaP /mnt/archive/gndnav/ /media/lvis/GROUND_NAV/gndnav/
 </code>
 ==== Care Package Creation ====
@@ Line 121: / Line 124: @@
 === Generate KMZs ===
+== On Ground If RTP Fails ==
   - Create directory structure and symbolically link RTP ql2 files from aircraft <code>
@@ Line 131: / Line 136: @@
   - Kick off processing <code>
 time ~/field_tools/local/rtp_local_proc.sh ~/field_tools/local/config_610i_arcsix2024.cfg ${mjd}
+</code>
+  - Channel mixed processing <code>
+time ~/field_tools/local/rtp_local_proc.sh ~/field_tools/local/config_610i_arcsix2024_chanmix.cfg ${mjd}
 </code>
@@ Line 155: / Line 163: @@
 rm files/*
 tiff2kml.py energy "$inst""$mjd"_Field_Energy.kmz "$inst""$mjd"_Field_Energy energy.png ${HOME}/field_tools/local/LVIS.png coords.txt
+</code>
+== Copy RTP Results ==
+This is the standard, only do the above if the onboard RTP generation didn't produce results
+  - Create a directory in the **proc** drive <code>
+mkdir -p /mnt/proc/lvisf/${mjd}
+</code>
+  - Copy the files from the RTP archive (which is a copy from the plane) (excluding rtlivs just in case it was somehow copied over) <code>
+rsync -vaP --exclude='rtlvis' /mnt/archive/rtp/mjd/${mjd}/ /mnt/proc/lvisf/${mjd}/
+</code>
+  - Copy all **kmz** files to external drive (optional, but a good idea assuming GROUND_NAV is mounted!) (max depth was because there is an older folder in one of the days to avoid copying those) <code>
+find /mnt/proc/lvisf/ -maxdepth 2 -name "*.kmz" -type f -exec cp -a {} /media/lvis/GROUND_NAV/kmz/ \;
 </code>
@@ Line 175: / Line 197: @@
 === Mission Meta ===
+<code>
+ln -s /mnt/archive/mjd/60410_60550_ancillary/mission_meta/coverage/arcsix2024.cfg ~/field_tools/mission_meta/
+</code>
   - Modify the configuration file so that the last mjd is captured
     - Edit the config <code>
-nano /home/lvis/field_tools/mission_meta/arcsix2024.cfg
+nano /mnt/archive/mjd/60410_60550_ancillary/mission_meta/coverage/arcsix2024.cfg
 </code>
   - Generate mission meta data <code>
 cd ~/field_tools/mission_meta
 time ./make_mission_data_summary.py arcsix2024.cfg
+</code>
+  - Copy to external drive
+    - Mount GROUND_NAV
+    - Sync the files to it <code>
+rsync -vaP /mnt/archive/mjd/60410_60550_ancillary/mission_meta/coverage/ /media/lvis/GROUND_NAV/coverage/
 </code>
@@ Line 191: / Line 222: @@
   - Attached external drive and mount with Ubuntu manager
   - Run this script based on MJD <code>
-sleep 2h ; create_mjd_external_archive.sh ${mjd} /media/lvis/A6_BACKUP02/
+sleep 2h ; create_mjd_external_archive.sh ${mjd} /media/lvis/A6_BACKUP03/
 </code>
+  - Copy the camera folder to the external drive.
+=== Copy Applanix Logs from External Camera Backup ===
+<code>
+find /media/lvis/A6_BACKUP04 -maxdepth 2 -type f -iname '*_log.txt'  -exec cp "{}" /media/lvis/GROUND_NAV/applanix_logs/ \;
+# to make sure you get ALL txt files (Rob and John named them differently)
+find /mnt/archive/camera/id/ -maxdepth 3 -type f -iname '*.txt'  -exec cp "{}" /tmp/applanix_logs/ \;
+</code>
+==== Raw Data Copy to Gaia ====
+  - Copy card data <code>
+rsync -vazP /mnt/archive/card/ lvis@gaia.lattice.net:incoming/card/
+</code>
+  - Copy usb data <code>
+rsync -vazP /mnt/archive/usb/ lvis@gaia.lattice.net:incoming/usb/
+</code>
+==== Verification ====
+  * Run md5deep on LVIS raw files
+  * Run teqc on GPS raw files
+  * Run applanix time check on raw IMU files
+  * Camera?  FIXME
+==== Final Raid Inventory ====
+  * Ensure each drive has a meta and inventory entry
+  * Copy inventory and meta files to three places **drive_meta/year/campaign/**
+    * camera, card, lvisf, nav, rtp, usb
+    * Box, Teams and Wiki <code>
+# manual creation of inventory
+time_date=$(date '+%Y%m%d')
+time_time=$(date '+%H%M%S')
+inventory_output="${camera_basepath}/inventory/${time_date}_${time_time}_${camera_diskid}_${linkname}_inventory.txt"
+meta_output="${camera_basepath}/meta/${linkname}.txt"
+mkdir -p ${camera_basepath}/inventory/ ${camera_basepath}/meta/
+meta_path="${camera_basepath}/meta/"
+drivemeta=$(df -h | grep ${camera_mountpoint})
+echo "${time_date} ${time_time} ${camera_diskid} ${drivemeta}"
+echo "${time_date} ${time_time} ${camera_diskid} ${drivemeta}" >> ${meta_output}
+echo "find ${camera_mountpoint} -type f -exec ls -l {} \; >> ${inventory_output}"
+find ${camera_mountpoint} -type f -exec ls -l {} \; >> ${inventory_output}
+</code>
+  * **md5deep** will follow symbolic links, so run this <code>
+time sudo find /mnt/archive/ -type f -exec md5deep -z {} \; > ~/md5deep_archive2024_all.txt
+# 68 hours for 44TB not bad
+# /dev/sda2        44T   43T  970G  98% /mnt/archive
+real    4033m5.615s
+user    1616m48.209s
+sys     429m25.964s
+# and make a sorted version of the file
+cat ~/md5deep_archive2024_all.txt  | sort -k 3 > ~/md5deep_archive2024_all_sorted.txt
+</code>