Skip to content

stream: improved support for large tag files #921

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ourairquality wants to merge 1 commit into
base: main
Choose a base branch
from
Open

stream: improved support for large tag files #921

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 58 additions & 37 deletions src/stream.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@
* use integer types in stdint.h
*-----------------------------------------------------------------------------*/
#define _POSIX_C_SOURCE 200112L

#ifndef WIN32
// For 64 bit file offsets on Linux and MacOS.
#define _FILE_OFFSET_BITS 64
#endif

#include <ctype.h>
#include "rtklib.h"
#ifndef WIN32
Expand Down Expand Up @@ -155,7 +161,7 @@ typedef struct { /* file control type */
gtime_t wtime; /* write time */
uint32_t tick; /* start tick */
uint32_t tick_f; /* start tick in file */
long fpos_n; /* next file position */
int64_t fpos_n; // Next file position
uint32_t tick_n; /* next tick */
double start; /* start offset (s) */
double speed; /* replay speed (time factor) */
Expand Down Expand Up @@ -724,7 +730,8 @@ static file_t *openfile(const char *path, int mode, char *msg)
file->offset=0;
file->size_fpos=size_fpos;
file->time=file->wtime=time0;
file->tick=file->tick_f=file->tick_n=file->fpos_n=0;
file->tick=file->tick_f=file->tick_n=0;
file->fpos_n=0;
file->start=start;
file->speed=speed;
file->swapintv=swapintv;
Expand Down Expand Up @@ -813,15 +820,29 @@ static int statexfile(file_t *file, char *msg)
sprintf(p," swapintv= %.3f\n",file->swapintv);
return state;
}

static int64_t xftell(FILE *stream)
{
#ifdef WIN32
return _ftelli64(stream);
#else
return ftello(stream);
#endif
}
static int xfseek(FILE *stream, int64_t offset, int whence)
{
#ifdef WIN32
return _fseeki64(stream, offset, whence);
#else
return fseeko(stream, offset, whence);
#endif
}

/* read file -----------------------------------------------------------------*/
static int readfile(file_t *file, uint8_t *buff, int nmax, char *msg)
{
struct timeval tv={0};
fd_set rs;
uint64_t fpos_8B;
uint32_t t,tick,fpos_4B;
long pos,n;
int nr=0;
uint32_t t,tick;

tracet(4,"readfile: fp=%d nmax=%d\n",file->fp,nmax);

Expand All @@ -830,10 +851,12 @@ static int readfile(file_t *file, uint8_t *buff, int nmax, char *msg)
if (file->fp==stdin) {
#ifndef WIN32
/* input from stdin */
fd_set rs;
FD_ZERO(&rs); FD_SET(0,&rs);
if (!select(1,&rs,NULL,NULL,&tv)) return 0;
if ((nr=(int)read(0,buff,nmax))<0) return 0;
return nr;
ssize_t nr = read(0, buff, nmax);
if (nr < 0) return 0;
return (int)nr;
#else
return 0;
#endif
Expand All @@ -850,18 +873,19 @@ static int readfile(file_t *file, uint8_t *buff, int nmax, char *msg)
}
/* seek time-tag file to get next tick and file position */
while ((int)(file->tick_n-t)<=0) {

uint32_t fpos_4B;
uint64_t fpos_8B;
if (fread(&file->tick_n,sizeof(tick),1,file->fp_tag)<1||
fread((file->size_fpos==4)?(void *)&fpos_4B:(void *)&fpos_8B,
file->size_fpos,1,file->fp_tag)<1) {
file->tick_n=(uint32_t)(-1);
pos=ftell(file->fp);
fseek(file->fp,0L,SEEK_END);
file->fpos_n=ftell(file->fp);
fseek(file->fp,pos,SEEK_SET);
int64_t pos = xftell(file->fp);
xfseek(file->fp, 0, SEEK_END);
file->fpos_n = xftell(file->fp);
xfseek(file->fp, pos, SEEK_SET);
break;
}
file->fpos_n=(long)((file->size_fpos==4)?fpos_4B:fpos_8B);
file->fpos_n = file->size_fpos == 4 ? (int64_t)fpos_4B : (int64_t)fpos_8B;
}
if (file->tick_n==(uint32_t)(-1)) {
sprintf(msg,"end");
Expand All @@ -871,29 +895,25 @@ static int readfile(file_t *file, uint8_t *buff, int nmax, char *msg)
file->wtime=timeadd(file->time,(int)t*0.001);
timeset(timeadd(gpst2utc(file->time),(int)file->tick_n*0.001));
}
if ((n=file->fpos_n-ftell(file->fp))<nmax) {
nmax=n;
}
int64_t n = file->fpos_n - xftell(file->fp);
if (n < nmax) nmax = (int)n;
}
size_t nr = 0;
if (nmax>0) {
nr=(int)fread(buff,1,nmax,file->fp);
}
if (feof(file->fp)) {
sprintf(msg,"end");
nr = fread(buff,1,nmax,file->fp);
}
tracet(5,"readfile: fp=%d nr=%d\n",file->fp,nr);
return nr;
if (feof(file->fp)) sprintf(msg,"end");
tracet(5,"readfile: fp=%d nr=%zu\n",file->fp,nr);
return (int)nr;
}
/* write file ----------------------------------------------------------------*/
static int writefile(file_t *file, uint8_t *buff, int n, char *msg)
{
gtime_t wtime;
uint64_t fpos_8B;
uint32_t tick=tickget(),fpos_4B;
int week1,week2,ns;
uint32_t tick=tickget();
int week1,week2;
double tow1,tow2,intv;
long fpos,fpos_tmp=0;


tracet(4,"writefile: fp=%d n=%d\n",file->fp,n);

if (!file) return 0;
Expand All @@ -918,43 +938,44 @@ static int writefile(file_t *file, uint8_t *buff, int n, char *msg)
}
if (!file->fp) return 0;

ns=(int)fwrite(buff,1,n,file->fp);
fpos=ftell(file->fp);
size_t ns = fwrite(buff, 1, n, file->fp);
int64_t fpos = xftell(file->fp);
int64_t fpos_tmp = 0;
fflush(file->fp);
file->wtime=wtime;

if (file->fp_tmp) {
fwrite(buff,1,n,file->fp_tmp);
fpos_tmp=ftell(file->fp_tmp);
fpos_tmp = xftell(file->fp_tmp);
fflush(file->fp_tmp);
}
if (file->fp_tag) {
tick-=file->tick;
fwrite(&tick,1,sizeof(tick),file->fp_tag);
if (file->size_fpos==4) {
fpos_4B=(uint32_t)fpos;
uint32_t fpos_4B = (uint32_t)fpos;
fwrite(&fpos_4B,1,sizeof(fpos_4B),file->fp_tag);
}
else {
fpos_8B=(uint64_t)fpos;
uint64_t fpos_8B = (uint64_t)fpos;
fwrite(&fpos_8B,1,sizeof(fpos_8B),file->fp_tag);
}
fflush(file->fp_tag);

if (file->fp_tag_tmp) {
fwrite(&tick,1,sizeof(tick),file->fp_tag_tmp);
if (file->size_fpos==4) {
fpos_4B=(uint32_t)fpos_tmp;
uint32_t fpos_4B = (uint32_t)fpos_tmp;
fwrite(&fpos_4B,1,sizeof(fpos_4B),file->fp_tag_tmp);
}
else {
fpos_8B=(uint64_t)fpos_tmp;
uint64_t fpos_8B = (uint64_t)fpos_tmp;
fwrite(&fpos_8B,1,sizeof(fpos_8B),file->fp_tag_tmp);
}
fflush(file->fp_tag_tmp);
}
}
tracet(5,"writefile: fp=%d ns=%d tick=%5d fpos=%d\n",file->fp,ns,tick,fpos);
tracet(5,"writefile: fp=%d ns=%zu tick=%5d fpos=%lld\n",file->fp,ns,tick,(long long)fpos);

return ns;
}
Expand Down