/* * $Id: vdifsup.c 4248 2017-02-28 22:37:57Z gbc $ * * This file provides support for the fuse interface. * This version is rather primitive in many respects. * This file does the work related to the metadata cache. */ #include #include #include #include #include #include #include "vdifuse.h" #include "sg_access.h" /* * Internal parameters that are preserved in the cache. * This temporary copy is for command-line adjustments, * then they are copied to the cache at the start. * * Note that during construction, the cache grows as needed * with realloc, so cache pointers should be considered dynamic, * but that the indices in the cache should be stable. */ static VDIFUSEpars clvpo = { .creation.tv_sec = 1, .station_mask = SG_STATION_MASK }; static char *filelistprefix = "."; #define VDIFUSE_CACHE_CHUNK 8 static char *vd_cache_file = NULL; static VDIFUSEntry *vd_cache = 0; static uint32_t vd_num_entries = 0; static uint32_t vd_num_alloced = 0; static int vd_cache_modified = 0; /* * Ok, time for real work */ static int accepted_as_frag(VDIFUSEntry *vc) { int rigor, cfe; VDIFUSEpars *vp = &vd_cache[0].u.vpars; if (vdifuse_debug>1) fprintf(vdflog, " Considering %s\n", vc->path); /* see if the file is acceptable as a fragment */ rigor = vdif_rigor_frag(vc->path, vp); /* if it actually passed the tests that matter */ if ((rigor & vp->how_rigorous) == vp->how_rigorous) { if ((cfe = create_fragment_vfuse(vc, vd_num_entries, vp, rigor))) {// || /* then we not able to accept this as a fragment */ vc->etype = VDIFUSE_ENTRY_INVALID; if (vdifuse_debug>2) fprintf(vdflog, " but failed [create:%d]\n", cfe); if (vdifuse_debug>0) fprintf(vdflog, "Problematic file %s [%d]\n", vc->path, cfe); } else { /* then it is officially added as a fragment */ vc->etype = VDIFUSE_ENTRY_FRAGMENT; if (vdifuse_debug>3) fprintf(vdflog, " Accepted %s\n" " " VDIFUSE_RIGOR_PRINTF " sub-type %u\n", vc->path, rigor, vc->stype); return(1); } } if (vdifuse_debug>2) fprintf(vdflog, " Rejected %s " VDIFUSE_RIGOR_PRINTF "\n", vc->path, rigor); return(0); } /* * Helper to extend the cache and zero the new space (to prevent accidents). * Upon entry vd_num_entries are assumed to be valid. * calloc works in objects, realloc works in bytes. */ static int extend_vd_cache(void) { size_t new_bytes, tot_bytes; VDIFUSEntry *old = vd_cache; VDIFUSEntry *new_cache; if (vd_num_alloced > (0xFFFFFFFEU - VDIFUSE_CACHE_CHUNK)) return(fprintf(stderr, "Cache limit exceeded\n")); vd_num_alloced += VDIFUSE_CACHE_CHUNK; new_bytes = VDIFUSE_CACHE_CHUNK * sizeof(VDIFUSEntry); tot_bytes = vd_num_alloced * sizeof(VDIFUSEntry); new_cache = (VDIFUSEntry *)realloc(vd_cache, tot_bytes); if (!new_cache) return(perror("realloc"),1); memset(new_cache + vd_num_entries, 0, new_bytes); vd_cache = new_cache; if (vdifuse_debug>2) fprintf(vdflog, "Extended cache to %d/%d entries (from %p to %p)\n", vd_num_entries, vd_num_alloced, old, new_cache); return(0); } /* * Convenience functions which let us access the cache from another * module. (Otherwise we'd have to put all the sequence stuff here.) * * We have vd_num_entries+1 to allow sg_access() to eventually be called * sooner. It doesn't hurt to add an extra entry. */ #define current_cache_full() ((vd_num_entries+1) >= vd_num_alloced) VDIFUSEntry *current_cache_start(void) { return(vd_cache); } uint32_t current_cache_entries(void) { return(vd_num_entries); } VDIFUSEntry *create_cache_entry(void) { if (current_cache_full() && extend_vd_cache()) return(NULL); vd_cache[vd_num_entries].index = vd_num_entries; vd_cache[vd_num_entries].etype = VDIFUSE_ENTRY_INVALID; vd_cache_modified++; return(&vd_cache[vd_num_entries++]); } VDIFUSEpars *current_cache_pars(void) { return(&(vd_cache[VDIFUSE_TOPDIR_PARAMS].u.vpars)); } /* defined below */ static int find_fragments(char *dir); /* * Support code for readability in find_fragments() */ static int handle_frag(char *file, char *parent) { char *tp; VDIFUSEntry *vt = create_cache_entry(); if (!vt) return(1); tp = vt->path; if (snprintf(tp, VDIFUSE_MAX_PATH, "%s/%s", parent, file) >= VDIFUSE_MAX_PATH) return(fprintf(stderr, "Path too long on %s/%s\n", parent, file)); if (accepted_as_frag(vt)) { /* only sgv2 requires an anc entry */ if (vt->stype == VDIFUSE_STYPE_SGV2) attach_sgv2_anc(vt->index); } else { vt->etype = VDIFUSE_ENTRY_INVALID; vd_num_entries--; } return(0); } static char *tmp_abspath(char *dir, char *parent) { char *abspath; int abslen; abslen = strlen(parent) + strlen(dir) + 2; if (abslen > VDIFUSE_MAX_PATH) { fprintf(stderr, "Path exceeded on %s/%s\n", parent, dir); return(NULL); } abspath = malloc(abslen + 1); if (!abspath) return(perror("malloc"),NULL); snprintf(abspath, abslen, "%s/%s", parent, dir); return(abspath); } static int handle_dir(char *dir, char *parent) { char *abspath; abspath = tmp_abspath(dir, parent); if (!abspath) return(1); if (find_fragments(abspath)) return(fprintf(stderr, "Problem with subdir %s/%s\n", parent, dir)); free(abspath); return(0); } /* * For filesystems that do not present the type in the dirent struct, * we need to use stat or lstat to figure out what to do. This version * uses stat (so it will ignore symbolic links) and passes the buck. */ static int handle_bogey(unsigned char type, char *dir, char *parent) { struct stat sb; char *abspath; int rv; abspath = tmp_abspath(dir, parent); if (!abspath) return(1); if (stat(abspath, &sb)) return(perror("stat"),2); if (S_ISREG(sb.st_mode)) { /* is it a regular file? */ rv = handle_frag(dir, parent); } else if (S_ISDIR(sb.st_mode)) { /* directory? */ rv = find_fragments(abspath); } else { if (vdifuse_debug>1) fprintf(vdflog, "Type %d on %s/%s mode %x ignored\n", type, parent, dir, sb.st_mode); rv = 2; } free(abspath); return(rv); } /* * Walk through a directory hierachy. For each entry, * ignore certain things * if a file: test it for vdif * if a dir: recurse into it * We assume here that we have enough stack space (which seems to be true). */ static int find_fragments(char *dir) { DIR *dp; struct dirent de, *dep; if (vdifuse_debug>1) fprintf(vdflog, "Searching %s\n", dir); dp = opendir(dir); if (!dp && errno == ENOTDIR) { if (vdifuse_debug>1) fprintf(vdflog, "which is not a dir\n"); return(0); } if (!dp) return(perror("opendir"),1); do { if (readdir_r(dp, &de, &dep)) return(perror("readdir_r"),2); if (dep == &de) { if (de.d_name[0] == '.') { continue; /* ignore hidden files, . and .. */ } else if (de.d_type == DT_REG) { if (handle_frag(de.d_name, dir)) return(fprintf(stderr, "Problem with fragment %s of %s\n", de.d_name, dir)); } else if (de.d_type == DT_DIR) { if (handle_dir(de.d_name, dir)) return(fprintf(stderr, "Problem with directory %s of %s\n", de.d_name, dir)); } else { if (handle_bogey(de.d_type, de.d_name, dir)) return(fprintf(stderr, "Problem with entry %s of %s\n", de.d_name, dir)); } } } while (dep); return(closedir(dp)); } /* * Cache creation has four steps: * first: find the fragments of vdif files * second: create a tree of fragments * third: link them together into sequences * fourth: finalize bookkeeping data */ static int populate_cache(int ndirs, char **dirs) { char *abspath; int had; if (vdifuse_debug>0) fprintf(vdflog, "Populating cache with %d directories\n", ndirs); while (ndirs-- > 0) { had = vd_num_entries; abspath = realpath(*dirs, 0); if (!abspath) return(fprintf(stderr, "No Abspath for %s\n", *dirs)); dirs++; if (find_fragments(abspath)) return(fprintf(stderr, "Error seaching %s\n", abspath)); if (vdifuse_debug>0) fprintf(vdflog, "Searched %s: %u -> %u entries\n", abspath, had, vd_num_entries); } if (create_fragtree()) return(fprintf(stderr, "Problem creating fragment tree\n")); if (create_sequences()) return(fprintf(stderr, "Problem creating sequences\n")); if (vdifuse_debug>0) fprintf(vdflog, "Constructed sequences, have %u entries\n", vd_num_entries); return(0); } static int create_params_entry(void) { int ee = VDIFUSE_TOPDIR_PARAMS; vd_cache[ee].index = ee; vd_cache[ee].etype = VDIFUSE_ENTRY_PARAMS; vd_cache[ee].cindex = vd_cache[ee].ccount = 0; /* command line supplied items */ vd_cache[ee].u.vpars.vdifuse_vers = VDIFUSE_VERSION; vd_cache[ee].u.vpars.prefix_bytes = clvpo.prefix_bytes; vd_cache[ee].u.vpars.offset_bytes = clvpo.offset_bytes; vd_cache[ee].u.vpars.searchwindow = clvpo.searchwindow; // nyi vd_cache[ee].u.vpars.catchbuserrs = clvpo.catchbuserrs; vd_cache[ee].u.vpars.max_pkts_gap = clvpo.max_pkts_gap; // nyi vd_cache[ee].u.vpars.max_secs_gap = clvpo.max_secs_gap; // nyi vd_cache[ee].u.vpars.how_rigorous = clvpo.how_rigorous; vd_cache[ee].u.vpars.noduplicates = clvpo.noduplicates; vd_cache[ee].u.vpars.seqhierarchy = clvpo.seqhierarchy; vd_cache[ee].u.vpars.writeblocker = clvpo.writeblocker; vd_cache[ee].u.vpars.pkts_per_sec = clvpo.pkts_per_sec; vd_cache[ee].u.vpars.dropfraction = clvpo.dropfraction; // nyi vd_cache[ee].u.vpars.station_mask = clvpo.station_mask; sg_set_station_id_mask(clvpo.station_mask & SG_STATION_MASK); /* topdir names for fragments and sequences */ if (clvpo.frag_top_dir[0] != '/') strcpy(clvpo.frag_top_dir, VDIFUSE_FRAGMENTS); strcpy(vd_cache[ee].u.vpars.frag_top_dir, clvpo.frag_top_dir); strcpy(fragments_topdir, clvpo.frag_top_dir); fragments_topdir_len = strlen(fragments_topdir); if (clvpo.seqs_top_dir[0] != '/') strcpy(clvpo.seqs_top_dir, VDIFUSE_SEQUENCES); strcpy(vd_cache[ee].u.vpars.seqs_top_dir, clvpo.seqs_top_dir); strcpy(sequences_topdir, clvpo.seqs_top_dir); sequences_topdir_len = strlen(sequences_topdir); /* discovered items */ vd_cache[ee].u.vpars.est_pkt_rate = 1; vd_cache[ee].u.vpars.maxfrcounter = 0; if (gettimeofday(&vd_cache[ee].u.vpars.creation, 0)) return(perror("gettimeofday"),1); /* not to match anything else, ever */ strcpy(vd_cache[ee].path, "--cache-parameters-entry-path--"); strcpy(vd_cache[ee].fuse, "--cache-parameters-entry-fuse--"); strcpy(vd_cache[ee].hier, "--cache-parameters-entry-hier--"); vd_cache_modified++; return(0); } /* * Should modify for m(first) c(final) a(delta) to agree with frag usage. */ void set_creation_time(struct stat *stbuf) { stbuf->st_mtime = stbuf->st_ctime = stbuf->st_atime = vd_cache[0].u.vpars.creation.tv_sec; stbuf->st_mtim.tv_nsec = stbuf->st_ctim.tv_nsec = stbuf->st_atim.tv_nsec = vd_cache[0].u.vpars.creation.tv_usec * 1000; } /* * For the following three methods, mp == NULL in initial cache * creation. If it is not null, we actually have a mount point * and we want to inherit almost all of the stbuf info from it. */ static int create_root_entry(struct stat *mp) { int ee = VDIFUSE_TOPDIR_ROOT_DIR; if (mp) { /* update directory with mountpoint stat data */ struct stat *stbuf = &vd_cache[ee].u.vfuse; *stbuf = *mp; stbuf->st_mode = S_IFDIR | 0555; stbuf->st_nlink = 4; } else { vd_cache[ee].index = ee; vd_cache[ee].etype = VDIFUSE_ENTRY_DIRECTORY; vd_cache[ee].cindex = vd_cache[ee].ccount = 0; strcpy(vd_cache[ee].fuse, "/"); vd_cache_modified++; } return(0); } static int create_fragments_entry(struct stat *mp) { int ee = VDIFUSE_TOPDIR_FRAGMENT; if (mp) { /* update directory with mountpoint stat data */ struct stat *stbuf = &vd_cache[ee].u.vfuse; *stbuf = *mp; stbuf->st_mode = S_IFDIR | 0555; stbuf->st_nlink = 2; } else { vd_cache[ee].index = ee; vd_cache[ee].etype = VDIFUSE_ENTRY_DIRECTORY; vd_cache[ee].cindex = vd_cache[ee].ccount = 0; strcpy(vd_cache[ee].fuse, fragments_topdir); vd_cache_modified++; } return(0); } static int create_sequences_entry(struct stat *mp) { int ee = VDIFUSE_TOPDIR_SEQUENCE; if (mp) { /* update directory with mountpoint stat data */ struct stat *stbuf = &vd_cache[ee].u.vfuse; *stbuf = *mp; stbuf->st_mode = S_IFDIR | 0555; stbuf->st_nlink = 2; } else { vd_cache[ee].index = ee; vd_cache[ee].etype = VDIFUSE_ENTRY_DIRECTORY; vd_cache[ee].cindex = vd_cache[ee].ccount = 0; strcpy(vd_cache[ee].hier, sequences_topdir); strcpy(vd_cache[ee].fuse, sequences_topdir); vd_cache_modified++; } return(0); } /* * Sequences and Directories inherit ownership from the mount point */ static int update_other_entries(struct stat *mp) { int ee; for (ee = VDIFUSE_TOPDIR_SEQUENCE+1; ee < vd_num_entries; ee++) { if (vd_cache[ee].etype != VDIFUSE_ENTRY_SEQUENCE && vd_cache[ee].etype != VDIFUSE_ENTRY_DIRECTORY) continue; if (vdifuse_debug>2) fprintf(vdflog, "%s: uid %d -> %d gid %d -> %d &c.\n", vd_cache[ee].fuse, vd_cache[ee].u.vfuse.st_uid, mp->st_uid, vd_cache[ee].u.vfuse.st_gid, mp->st_gid); vd_cache[ee].u.vfuse.st_uid = mp->st_uid; vd_cache[ee].u.vfuse.st_gid = mp->st_gid; } return(0); } /* * Allocate memory for the working cache. * This version just callocs a starter and (later) writes it out. */ static int create_cache(char *cache) { vd_num_alloced = VDIFUSE_CACHE_CHUNK; vd_cache = (VDIFUSEntry*)calloc(vd_num_alloced, sizeof(VDIFUSEntry)); if (!vd_cache) return(perror("calloc"),1); /* no reason not to put params entry first */ if (create_params_entry()) return(2); if (create_root_entry(NULL)) return(3); if (create_fragments_entry(NULL)) return(4); if (create_sequences_entry(NULL)) return(5); vd_num_entries = VDIFUSE_TOPDIR_SEQUENCE + 1; vd_cache_file = cache; return(0); } static int load_cache(char *cache) { int ee = VDIFUSE_TOPDIR_PARAMS; struct stat sb; FILE *fp; size_t ne; if (vd_num_entries && vd_cache && vd_cache_file) return(0); if (stat(cache, &sb)) return(perror("stat"),1); vd_num_entries = sb.st_size / sizeof(VDIFUSEntry); vd_num_alloced = vd_num_entries; vd_cache = (VDIFUSEntry*)calloc(vd_num_entries, sizeof(VDIFUSEntry)); if (!vd_cache) return(perror("calloc"),1); fp = fopen(cache, "r"); if (!fp) return(fprintf(stderr,"%s: ", cache), perror("load_cache fopen"),2); ne = fread(vd_cache, sizeof(VDIFUSEntry), vd_num_entries, fp); if (ne != vd_num_entries) return(fprintf(stderr, "Failed to load complete cache\n")); vd_cache_file = cache; strcpy(fragments_topdir, vd_cache[ee].u.vpars.frag_top_dir); strcpy(sequences_topdir, vd_cache[ee].u.vpars.seqs_top_dir); fragments_topdir_len = strlen(fragments_topdir); sequences_topdir_len = strlen(sequences_topdir); sg_set_station_id_mask(vd_cache[ee].u.vpars.station_mask & SG_STATION_MASK); return(fclose(fp)); } static int dump_cache(void) { FILE *fp; size_t ne; if (!vd_cache || vd_num_entries == 0 || !vd_cache_file) return(0); /* no work */ fp = fopen(vd_cache_file, "w"); if (!fp) return(fprintf(stderr,"%s: ",vd_cache_file), perror("dump_cache fopen"),1); ne = fwrite(vd_cache, sizeof(VDIFUSEntry), vd_num_entries, fp); if (ne != vd_num_entries) return(fprintf(stderr, "Failed to write complete cache\n")); if (vdifuse_debug>0) fprintf(vdflog, "Wrote cache to %s (%d mods)\n", vd_cache_file, vd_cache_modified); return(fclose(fp)); } /* * Create the VDIF meta-data cache. Return 0 if the cache * created is ready for use, otherwise return nonzero. */ int vdifuse_create_metadata(char *cache, int ndirs, char **dirs) { int err; if (vdifuse_debug>0) fprintf(vdflog, "Creating Meta Data cache for fuse in %s\n", cache); if ((err = create_cache(cache))) return(fprintf(stderr, "Unable to create cache %s (%d)\n", cache, err)); if ((err = populate_cache(ndirs, dirs))) return(fprintf(stderr, "Unable to build cache with the %d directories provided (%d)\n", ndirs, err)); /* at this point the cache is created, and in memory */ return(0); } /* * stat buf usage for fragments: * (normal usage except for + items) * + dev_t st_dev; // prefix_bytes (to first packet) * + ino_t st_ino; // pkts_per_sec * mode_t st_mode; // protection * nlink_t st_nlink; // number of hard links * uid_t st_uid; // user ID of owner * gid_t st_gid; // group ID of owner * + dev_t st_rdev; // offset_bytes (VDIF hdr in packet) * off_t st_size; // total size, in bytes * + blksize_t st_blksize; // packet size * + blkcnt_t st_blocks; // number of packets * + time_t st_mtime; // data start time * + time_t st_ctime; // data end time (inc. last packet) * + time_t st_atime; // total scan duration */ int describe_fragment(VDIFUSEntry *vp) { if (vdifuse_debug>1) fprintf(vdflog, "[%04d]Frg %s\n", vp->index, vp->path); if (vdifuse_debug>2) fprintf(vdflog, " (index=%u vsig=%016lX)\n" " (size=%lu pktsize=%ld pkts=%ld prefix=%lu offset=%lu rate=%lu)\n" " (from=%lu.%09lu to=%lu.%09lu dur=%lu.%09lu)\n" " (hier %s)\n (frag %s)\n", vp->index, vp->vsig, vp->u.vfuse.st_size, vp->u.vfuse.st_blksize, vp->u.vfuse.st_blocks, vp->u.vfuse.st_dev, vp->u.vfuse.st_rdev, vp->u.vfuse.st_ino, vp->u.vfuse.st_mtime, vp->u.vfuse.st_mtim.tv_nsec, vp->u.vfuse.st_ctime, vp->u.vfuse.st_ctim.tv_nsec, vp->u.vfuse.st_atime, vp->u.vfuse.st_atim.tv_nsec, vp->hier, vp->fuse ); return(0); } static void cascade_time(struct tm *tm) { int din = (tm->tm_year / 4) ? 365 : 366; /* broken for 2000 */ if (tm->tm_sec == 60) { tm->tm_sec = 0; tm->tm_min ++; } if (tm->tm_min == 60) { tm->tm_min = 0; tm->tm_hour ++; } if (tm->tm_hour == 24) { tm->tm_hour = 0; tm->tm_yday ++; } if (tm->tm_yday == din) { tm->tm_yday = 0 ; tm->tm_year ++; } } int report_sequence(VDIFUSEntry *vp) { struct tm start; struct tm stop; /* round out to nearest second for this */ gmtime_r(&vp->u.vfuse.st_mtime, &start); if (vp->u.vfuse.st_mtim.tv_nsec > VDIFUSE_ONE_SEC_NS/2) start.tm_sec ++; cascade_time(&start); gmtime_r(&vp->u.vfuse.st_ctime, &stop); if (vp->u.vfuse.st_ctim.tv_nsec > VDIFUSE_ONE_SEC_NS/2) stop.tm_sec ++; cascade_time(&stop); fprintf(vdflog, "%s%-38s %04dy%03dd%02dh%02dm%02ds %04dy%03dd%02dh%02dm%02ds\n", filelistprefix, vp->fuse, start.tm_year + 1900, start.tm_yday + 1, start.tm_hour, start.tm_min, start.tm_sec, stop.tm_year + 1900, stop.tm_yday + 1, stop.tm_hour, stop.tm_min, stop.tm_sec); return(0); } int describe_sequence(VDIFUSEntry *vp) { if (vdifuse_debug>1) fprintf(vdflog, "[%04d]Seq %s anc [%d] %lu links %luB\n", vp->index, vp->fuse, vp->cindex, vp->u.vfuse.st_nlink, vp->u.vfuse.st_size); if (vdifuse_debug>2) fprintf(vdflog, " pktsize=%lu pkts=%lu %lu.%09lu %lu.%09lu %lu.%09lu\n", vp->u.vfuse.st_blksize, vp->u.vfuse.st_blocks, vp->u.vfuse.st_mtime, vp->u.vfuse.st_mtim.tv_nsec, vp->u.vfuse.st_ctime, vp->u.vfuse.st_ctim.tv_nsec, vp->u.vfuse.st_atime, vp->u.vfuse.st_atim.tv_nsec ); return(0); } int describe_directory(VDIFUSEntry *vp) { if (vdifuse_debug>1) fprintf(vdflog, "[%04d]Dir %s anc [%d]\n", vp->index, vp->fuse, vp->cindex); return(0); } int describe_struct(void) { fprintf(vdflog, "Size: The cache consists of entries of size %ld including a\n" "Size: union of size %ld (vfuse=%ld vpars=%ld vseqi=%ld voids=%ld),\n" "Size: paths of no more than %d bytes and sgv2 data (size=%d)\n" "Size: 6*4 + 8 + %ld + 3*%d = %ld\n", sizeof(VDIFUSEntry), sizeof(union vdifuse_union), sizeof(struct stat), sizeof(VDIFUSEpars), sizeof(uint32_t)*VDIFUSE_MAX_SEQI, sizeof(void *)*VDIFUSE_NUM_VOIDS, VDIFUSE_MAX_PATH, sg_info_size(), sizeof(union vdifuse_union), VDIFUSE_MAX_PATH, 6*4 + 8 + sizeof(union vdifuse_union) + 3*VDIFUSE_MAX_PATH); return(0); } int describe_params(VDIFUSEntry *vp) { time_t birth = vp->u.vpars.creation.tv_sec; if (vdifuse_debug>2) describe_struct(); if (vdifuse_debug>1) fprintf(vdflog, "[%04d]Cache (vers.%g) %lu.%06lu (%24.24s)\n", vp->index, vp->u.vpars.vdifuse_vers, vp->u.vpars.creation.tv_sec, vp->u.vpars.creation.tv_usec, ctime(&birth)); if (vdifuse_debug>2) fprintf(vdflog, " prefix_bytes=%u offset_bytes=%u searchwindow=%u\n" " bus_errors=%u station_mask=%x\n" " max_pkts_gap=%u max_secs_gap=%f " "how_rigorous=" VDIFUSE_RIGOR_PRINTF "\n" " noduplicates=%u seqhierarchy=%u writeblocker=%u\n" " pkts_per_sec=%u dropfraction=%f maxfrcounter=%u\n" " est_pkt_rate=%u frag_top_dir=%s seqs_top_dir=%s\n" , vp->u.vpars.prefix_bytes, vp->u.vpars.offset_bytes, vp->u.vpars.searchwindow, // nyi vp->u.vpars.catchbuserrs, vp->u.vpars.station_mask, vp->u.vpars.max_pkts_gap, // nyi vp->u.vpars.max_secs_gap, // nyi vp->u.vpars.how_rigorous, vp->u.vpars.noduplicates, vp->u.vpars.seqhierarchy, vp->u.vpars.writeblocker, vp->u.vpars.pkts_per_sec, vp->u.vpars.dropfraction, // nyi vp->u.vpars.maxfrcounter, vp->u.vpars.est_pkt_rate, vp->u.vpars.frag_top_dir, vp->u.vpars.seqs_top_dir ); return(0); } char *describe_stype(int stype) { char *anctype; switch (stype) { case VDIFUSE_STYPE_NULL: anctype = "Empty fragment"; break; case VDIFUSE_STYPE_VDIF: anctype = "VDIF fragment"; break; case VDIFUSE_STYPE_SGV2: anctype = "SGv2 fragment"; break; case VDIFUSE_STYPE_INFO: anctype = "SGv2 access"; break; case VDIFUSE_STYPE_SDIR: anctype = "Dir entries"; break; case VDIFUSE_STYPE_PART: anctype = "Part of seq"; break; default: fprintf(stderr, "Illegal stype %d\n", stype); anctype = "ERROR"; break; } return(anctype); } int describe_ancillary(VDIFUSEntry *vp) { char *anctype, *prefix, *suffix; int ee; anctype = describe_stype(vp->stype); if (vdifuse_debug>1) fprintf(vdflog, "[%04d]Anc %s (%s) with %d\n", vp->index, vp->fuse, anctype, vp->ccount); if (vdifuse_debug>2) { if (vp->stype == VDIFUSE_STYPE_INFO) return(describe_ancillary_sgv2(vp)); for (ee = 0; ee < vp->ccount; ee++) { prefix = (ee%8 == 0) ? " (" : ""; suffix = (ee%8 == 7) ? ")\n" : (ee == vp->ccount-1) ? ")\n" : " "; fprintf(vdflog, "%s[%04d]%s", prefix, vp->u.vseqi[ee], suffix); } } return(0); } /* * A debugging version of the guts of the following. */ void describe_cache_entry(VDIFUSEntry *vp) { switch (vp->etype) { case VDIFUSE_ENTRY_FRAGMENT: (void)describe_fragment(vp); break; case VDIFUSE_ENTRY_SEQUENCE: (void)describe_sequence(vp); break; case VDIFUSE_ENTRY_DIRECTORY: (void)describe_directory(vp); break; case VDIFUSE_ENTRY_PARAMS: (void)describe_params(vp); break; case VDIFUSE_ENTRY_ANCILLARY: (void)describe_ancillary(vp); break; default: case VDIFUSE_ENTRY_INVALID: fprintf(vdflog, "Entry (%p) marked invalid\n", vp); break; } } /* * Walk through the list, check and comment as required. */ static int report_on_cache(void) { long ee; long frags = 0, seqs = 0, dirs = 0, params = 0, anc = 0, invalids = 0; VDIFUSEntry *vp; for (ee = 0, vp = vd_cache; ee < vd_num_entries; ee++, vp++) { switch (vp->etype) { case VDIFUSE_ENTRY_FRAGMENT: frags++; if (describe_fragment(vp)) return(1); break; case VDIFUSE_ENTRY_SEQUENCE: seqs++; if (describe_sequence(vp)) return(2); break; case VDIFUSE_ENTRY_DIRECTORY: dirs++; if (describe_directory(vp)) return(3); break; case VDIFUSE_ENTRY_PARAMS: params++; if (describe_params(vp)) return(4); break; case VDIFUSE_ENTRY_ANCILLARY: anc++; if (describe_ancillary(vp)) return(5); break; default: case VDIFUSE_ENTRY_INVALID: invalids++; fprintf(vdflog, "Entry %ld (%s) marked invalid\n", ee, vp->path); break; } } if (vdifuse_debug>0) fprintf(vdflog, "Have %lu frags %lu seqs %lu dirs %lu params %lu anc %lu invalids\n", frags, seqs, dirs, params, anc, invalids); fflush(vdflog); return(0); } /* * A slightly different reporting function */ static int filelist_from_cache(void) { long ee; VDIFUSEntry *vp; for (ee = 0, vp = vd_cache; ee < vd_num_entries; ee++, vp++) { /* we only care about this one */ if ((vp->etype == VDIFUSE_ENTRY_SEQUENCE) && report_sequence(vp)) return(2); } fflush(vdflog); return(0); } static int check_topdirs(void) { char *x; x = vd_cache[VDIFUSE_TOPDIR_PARAMS].u.vpars.frag_top_dir; if (strcmp(x, fragments_topdir)) return(fprintf(stderr, "%s != %s\n", x, fragments_topdir)); x = vd_cache[VDIFUSE_TOPDIR_PARAMS].u.vpars.seqs_top_dir; if (strcmp(x, sequences_topdir)) return(fprintf(stderr, "%s != %s\n", x, sequences_topdir)); return(0); } /* * Backward compatibility on the cache is difficult, and it is * safer to just require the user to regenerate it. */ static int check_cache_version(void) { if (check_topdirs()) return(2); if (vd_cache[0].u.vpars.vdifuse_vers == (float)VDIFUSE_VERSION) return(0); fprintf(stderr, "File version %f != code version %f (%e)\n", vd_cache[0].u.vpars.vdifuse_vers, (float)VDIFUSE_VERSION, vd_cache[0].u.vpars.vdifuse_vers - (float)VDIFUSE_VERSION); fprintf(stderr, "You need to regenerate the file cache (-c)\n"); return(1); } /* * Check the cache file which is (at this point) known to exist * and contain at least one entry. Return 0 if the cache * passes some sanity checks. Return nonzero otherwise. * * If vdifuse_debug is nonzero, provide a verbose report. */ int vdifuse_report_metadata(char *cache, struct stat *sb) { int err; if (vdifuse_debug>0) fprintf(vdflog, "Reporting Meta Data cache '%s' for use with fuse.\n", cache); if (vdifuse_debug>0) fprintf(vdflog, "This cache contains %ld entries of size %ld\n", sb->st_size / sizeof(VDIFUSEntry), sizeof(VDIFUSEntry)); if ((err = load_cache(cache))) return(fprintf(stderr, "Unable to open cache '%s' (%d)\n", cache, err)); if (check_cache_version()) return(fprintf(stderr, "Incompatible cache '%s' -- cannot continue.\n", cache)); /* at this point the cache is in memory */ if (vd_num_entries && vd_cache) return(report_on_cache()); return (fprintf(stderr, "Cache wasn't loaded properly\n")); } /* * A variant of the previous, which generates filelists suitable * for use as filelists in v2d files. It is relatively terse. */ int vdifuse_report_filelist(char *cache, struct stat *sb) { int err; if ((err = load_cache(cache))) return(fprintf(stderr, "Unable to open cache '%s' (%d)\n", cache, err)); if (check_cache_version()) return(fprintf(stderr, "Incompatible cache '%s' -- cannot continue.\n", cache)); /* at this point the cache is in memory */ if (vd_num_entries && vd_cache) return(filelist_from_cache()); return (fprintf(stderr, "Cache wasn't loaded properly\n")); } /* * Update the cache with data from the mount point */ static int update_mount_point(struct stat *mp) { set_creation_time(mp); if (create_root_entry(mp) || create_fragments_entry(mp) || create_sequences_entry(mp) || update_other_entries(mp)) return(fprintf(stderr, "Issues with mount point\n")); return(0); } /* * If not creating or reporting, we're using */ int vdifuse_access_metadata(char *cache, struct stat *sb, struct stat *mp) { int err; /* check if already loaded */ if (vd_num_entries && vd_cache) return(update_mount_point(mp)); if (vdifuse_debug>0) fprintf(vdflog, "Accessing Meta Data cache for fuse in %s\n", cache); if (vdifuse_debug>0) fprintf(vdflog, "Cache contains %ld entries of size %ld\n", sb->st_size / sizeof(VDIFUSEntry), sizeof(VDIFUSEntry)); if ((err = load_cache(cache))) return(fprintf(stderr, "Unable to open cache %s (%d)\n", cache, err)); if (check_cache_version()) return(fprintf(stderr, "Incompatible cache %s -- cannot continue.\n", cache)); /* at this point the cache is in memory */ if (vd_num_entries && vd_cache) return(update_mount_point(mp)); return (fprintf(stderr, "Cache wasn't loaded properly\n")); } /* * Top-level directories are somewhat special. */ void vdifuse_topdir(int which, struct stat *stbuf) { switch (which) { case VDIFUSE_TOPDIR_ROOT_DIR: *stbuf = vd_cache[1].u.vfuse; break; case VDIFUSE_TOPDIR_FRAGMENT: *stbuf = vd_cache[2].u.vfuse; break; case VDIFUSE_TOPDIR_SEQUENCE: *stbuf = vd_cache[3].u.vfuse; break; default: fprintf(stderr, "Developer Error\n"); break; } } /* * Support for attributes; these return 0 if the path is found. * This lookup isn't terribly efficient, but computers are fast */ int vdifuse_fragment(const char *path, struct stat *stbuf) { int ii; for (ii = 0; ii < vd_num_entries; ii++) { /* this directory tree contains only fragments */ if (vd_cache[ii].etype != VDIFUSE_ENTRY_FRAGMENT) continue; if (!strcmp(path, vd_cache[ii].fuse)) { if (vdifuse_debug>1) fprintf(vdflog, "Found %s\n", path); memcpy(stbuf, &vd_cache[ii].u.vfuse, sizeof(struct stat)); return(0); } } if (vdifuse_debug>1) fprintf(vdflog, "Did not find %s\n", path); return(1); } int vdifuse_sequence(const char *path, struct stat *stbuf) { int ii; for (ii = 0; ii < vd_num_entries; ii++) { /* this directory tree contains subdirs and sequences */ if (vd_cache[ii].etype != VDIFUSE_ENTRY_DIRECTORY && vd_cache[ii].etype != VDIFUSE_ENTRY_SEQUENCE) continue; if (!strcmp(path, vd_cache[ii].fuse)) { if (vdifuse_debug>1) fprintf(vdflog, "Found %s\n", path); memcpy(stbuf, &vd_cache[ii].u.vfuse, sizeof(struct stat)); return(0); } } if (vdifuse_debug>1) fprintf(vdflog, "Did not find %s\n", path); return(1); } /* * Support for readdir. * Starting with index, provide the fuse name and stbuf data. * If no more, return 0. */ int get_vdifuse_fragment(int ii, char **name, struct stat **stbuf) { for (ii++ ; ii < vd_num_entries; ii++) { if (vd_cache[ii].etype != VDIFUSE_ENTRY_FRAGMENT) continue; *name = strrchr(vd_cache[ii].fuse, '/'); if (!*name) *name = "/.damaged-fragment"; (*name) ++ ; *stbuf = &vd_cache[ii].u.vfuse; return(ii); } return(0); /* no more */ } /* * Support for readdir. * Provide the top-level sequences directory listing. * There should be an ANCILLARY file with the info for the entries. * if seqhierarchy is 0, we're actually providing a list of sequences. * if seqhierarchy > 0, we're providing exp subdirs * Returns of 0 indicate we are done, otherwise return the * nonzero index of the subdirectory or sequence entry. */ int get_vdifuse_sequence(int ii, char **name, struct stat **stbuf) { static uint32_t sndx, sdir; static VDIFUSEntry *vseq; VDIFUSEntry *vc; if (ii == 0) { /* first time through */ vseq = &vd_cache[vd_cache[VDIFUSE_TOPDIR_SEQUENCE].cindex]; sdir = vseq->u.vseqi[sndx = 0]; } else { /* pick up where we left off */ if (++sndx == VDIFUSE_MAX_SEQI) { if (vseq->cindex == 0) return(0); vseq = &vd_cache[vseq->cindex]; sndx = 0; } sdir = vseq->u.vseqi[sndx]; } if (sdir == 0) { ii = 0; /* no more entries */ } else { vc = &vd_cache[sdir]; *name = strrchr(vc->fuse, '/'); if (!*name) *name = "/.damaged-sequence"; (*name) ++; *stbuf = &vc->u.vfuse; ii = vseq - vd_cache; /* index of current entry */ } return(ii); } /* * Return the index of the sequence subdirectory matching path, if found. * If found, subsequence calls to get_vdifuse_subdir() fill in the entries. * This pair is similar to get_vdifuse_sequence(), except that the first * is needed to locate the parent subdirectory so that the second can * unload starting there. */ int get_sequence_subdir(const char *path) { int ii; for (ii = 0 ; ii < vd_num_entries; ii++) { if (vd_cache[ii].etype != VDIFUSE_ENTRY_SEQUENCE && vd_cache[ii].etype != VDIFUSE_ENTRY_DIRECTORY) continue; if (strcmp(path, vd_cache[ii].fuse)) continue; if (vdifuse_debug>1) fprintf(vdflog, "Found %s entry %d\n", path, ii); return(ii); } return(0); } int get_vdifuse_subdir(int ii, char **name, struct stat **stbuf) { static uint32_t sndx, sdir; static VDIFUSEntry *vseq; VDIFUSEntry *vc; if (vd_cache[ii].etype == VDIFUSE_ENTRY_SEQUENCE || vd_cache[ii].etype == VDIFUSE_ENTRY_DIRECTORY) { /* first time */ if (vd_cache[ii].cindex == 0) return(0); vseq = &vd_cache[vd_cache[ii].cindex]; sdir = vseq->u.vseqi[sndx = 0]; } else { if (++sndx == VDIFUSE_MAX_SEQI) { /* pick up where we left off */ if (vseq->cindex == 0) return(0); vseq = &vd_cache[vseq->cindex]; sndx = 0; } sdir = vseq->u.vseqi[sndx]; } if (sdir == 0) { ii = 0; /* no more entries */ } else { vc = &vd_cache[sdir]; *name = strrchr(vc->fuse, '/'); if (!*name) *name = "/.damaged-sequence"; (*name) ++; *stbuf = &vc->u.vfuse; ii = vseq - vd_cache; /* index of current entry */ } return(ii); } /* support for open, release, and read moved to vdirorr.c */ /* * Show the warts */ static int vdifuse_issues(void) { fprintf(vdflog, "Single-threaded VDIF is expected.\n"); fprintf(vdflog, "Multi-threaded VDIF is not currently supported.\n"); fprintf(vdflog, "A single data rate is expected and must be\n"); fprintf(vdflog, " supplied for good results: i.e. -xrate=pkts/sec.\n"); fprintf(vdflog, "Many cases of data corruption are handled, but\n"); fprintf(vdflog, " seriously corrupt data is likely to be tossed.\n"); fprintf(vdflog, "\n"); return(1); } /* * Yet more help */ static int vdifuse_examples(void) { fprintf(vdflog, "With single-threaded Mark6 scatter-gather files,\n"); fprintf(vdflog, "\n"); fprintf(vdflog, " vdifuse -a