/* main.c -- main entry point (including MPI initialization)
* 
*  This is also a convenient place to hold tile i/o routines called by cm.f
*/

#ifdef WIN32
#include <windows.h>
#include <winbase.h>
#include <io.h>
#include <fcntl.h>
#include <sys/types.h>
#else
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <sys/resource.h>
#endif

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <sys/types.h>

#include "mpi.h"
#include "thing.h"

static int Myid, Nodes;

#ifndef WIN32
#define TM tm_
#define CM cm_
#define WRITE_STUFF write_stuff_
#define READ_STUFF read_stuff_
#define CREATE_TILE create_tile_
#define OPENTRACE opentrace_
#define CLOSETRACE closetrace_
#define WRITETRACE writetrace_
#define SLEEP sleep_
#define Sleep(n) usleep(n*10);
#define MKDIR mkdir_
#define CDDONE cddone_
#define STRIPOUT stripout_
#define SETSTACK setstack_
#endif

void TM( int *node );
void CM( int *node );

#include "rdparams.h"

/* G L O B A L S */

/* These names are suffixes of things for each portion of a context.
* The writename order switching accomplishes the change from source
* to destination naming - "my xl", oo the neighbor's "xr" for example
*/

	static char *readName[9]= { "ddd", "bxl", "bxr", "byb", "byt",
		"bxlyb", "bxlyt", "bxryt", "bxryb" };
	static char *writeName[9]={ "ddd", "bxr", "bxl", "byt", "byb",
		"bxryt", "bxryb", "bxlyb", "bxlyt" };

/*--------------------------------------------------------------------------
*/

int main(int argc, char **argv) {

    int sharedFileSystem= 0;
    int myargc, ierr;
	char **myargv;

	fprintf(stderr,"main engine start..\n");

	/* we're going to call MPI here instead of within start_thing */
	myargc= 0;  myargv= NULL;
	MPI_Init( &argc, &argv );

	MPI_Comm_size( MPI_COMM_WORLD, &Nodes );
	MPI_Comm_rank( MPI_COMM_WORLD, &Myid );

	/* 	fprintf(stderr,"*** startup, node %d of %d\n", Myid+1, Nodes ); */

	if ( Myid == 0 ) {
		/* Attempt to read the 'globals' file in the file system */
		rdparams();
	}

	MPI_Bcast( &globals, sizeof(globals), MPI_CHAR, 0, MPI_COMM_WORLD );

	ierr= start_thing( globals.rootDir, sharedFileSystem, &myargc, &myargv );
	if ( ierr == -1 ) { perror("Could not start things"); exit(1); }

    stuffincommon_( &globals.restart, globals.runName, globals.rootDir, globals.globalsName, 
		    globals.homeDir, globals.xxl, globals.yyl, globals.zzl );

	if ( Myid == 0 ) list_thing();

	Nodes= getNodeCount_thing();

        fprintf(stderr,"%d Through start_thing \n", Myid );

	if ( Myid == 0 ) TM( &Myid );
	else CM( &Myid );

	stop_thing();
	exit(0);
	return 0;
}

/*---------------------------------------------------------------------
* Handy routines to construct the names of things.  Since we need two
* copies, it creates:
*
* tile interior:  0-XY-ddd  1-XY-ddd 2-xy'-ddd 3-xy'-ddd
* boundaries similarily, named bxl, bxr, byb, byt, bxlyb, bxryb, bxlyt, bxryt
* Each created name will have 10 characters (plus the terminating \0)
*/
void CREATE_TILE( int *task, unsigned *maxsize ) {
  char name[256], str[4][256];
  int i,j, itx, ity, jty;

	itx= task[0];
	ity= task[1];
	jty= task[3];
	sprintf(str[0], "0-%02d%02d-", itx, ity );
	sprintf(str[1], "1-%02d%02d-", itx, ity );
	sprintf(str[2], "2-%02d%02d-", itx, jty );
	sprintf(str[3], "3-%02d%02d-", itx, jty );

	/*    fprintf(stderr,"cm%d: in create_tile %d,%d maxizes are:\n",Myid,itx,ity); */

	for (j=0; j<2; ++j )
	  for (i=0; i<9; ++i ) {
	    strcpy(name,str[j] );
	    strcat(name,readName[i] );
	    /*	fprintf(stderr,"cm%d ------ %s size %d\n", Myid,name ,maxsize[i]*sizeof(float) ); */
	    if ( create_thing( name, DISK_THING, maxsize[i]*sizeof(float) ) == -1 ) perror(name);
	}

	/* Only create [2-3]-xy-ddd entries, no boundaries */

	for (j=2; j<4; ++j ) {
	  strcpy(name,str[j] );
	  strcat(name,readName[0] );
	  if ( create_thing( name, DISK_THING, maxsize[0]*sizeof(float) ) == -1 ) perror(name);
	}

	/* fprintf(stderr,"\n\n"); */

	return;
}

/*---------------------------------------------------------------------
* Starts reads of the interior zone & boundary elements
* data:		1.. name of the chunk (see static readName[] defined above)
* itx,ity	tile x,y (1..) for the 2D decomposition, coded into names
* iset		dataset 0 or 1
*
* So the encodedthing name is:   (set)-(tilex)(tiley)-writeName[data-1]
*
* zoffset	offset (reals) to start read
* nreals	how many to read
* status	resulting read_thing status to check
*/

void READ_STUFF( int *data, int *itx, int *ity, int *iset, int *zoffset, 
			     int *nreals, void *array, thing_t *status ) {

	char name[256];
	unsigned long offset;
	unsigned nbytes;
	int ierr, i;

    /* fprintf(stderr,"read_stuff zoffset,nreals= %d %d\n", *zoffset, *nreals ); */

	offset= sizeof(float)* *zoffset;
	nbytes= sizeof(float)* *nreals;

	if ( *iset <0 || *iset > 1 ) {
		fprintf(stderr,"Error: bad iset (%d) given read_bdry\n", *iset );
		return;
	}

	i= *data - 1;
	if ( i<0 || i>8 ) {
		fprintf(stderr,"Error: bad data set %d passed read_bdry\n", i );
		return;
	}

	sprintf(name,"%d-%02d%02d-%s", *iset, *itx, *ity, readName[i] );

	ierr= read_thing(name,offset, array, nbytes, status );
	if ( ierr == -1 ) 
		fprintf(stderr,"Error: could not read_thing '%s' offset %ud, bytes %ud\n",
			name, offset, nbytes );

	return;
}


/*----------------------------------------------------------------------
* Starts writes of the interior zone & boundary elements
* data:		1.. name of the chunk (see static writeName[] defined above)
* itx,ity	tile x,y (1..) for the 2D decomposition, coded into names
* iset		dataset 0 or 1
*
* So the encodedthing name is:   (set)-(tilex)(tiley)-writeName[data-1]
*
* zoffset	offset (reals) to start write
* nreals	how many to write
* status	resulting write_thing status to check
*/

void WRITE_STUFF( int *data, int *itx, int *ity, int *iset, int *zoffset,
				int *nreals, void *array, thing_t *status ) {

	char name[256];
	unsigned long offset;
	unsigned nbytes;
	int ierr, i;

	offset= sizeof(float)* *zoffset;
	nbytes= sizeof(float)* *nreals;

	if ( *iset <0 || *iset > 3 ) {
		fprintf(stderr,"Error: bad iset (%d) given write_bdry\n", *iset );
		return;
	}

	i= *data - 1;
	if ( i<0 || i>10 ) {
		fprintf(stderr,"Error: bad data set %d passed write_bdry\n", i );
		return;
	}
    if ( i > 8 ) i= 0;

	sprintf(name,"%d-%02d%02d-%s", *iset, *itx, *ity, writeName[i] );

	ierr= write_thing(name, offset, array, nbytes, status );
	if ( ierr == -1 ) 
		fprintf(stderr,"Error: could not write_thing '%s' offset %ud, bytes %ud\n",
			name, offset, nbytes );

	return;
}


/*-----------------------------------------------------------------------------
* Returns 0 if the globals file does not exist (with no messages)
*           Abort if it exists but is too short
*         1 if it exists and replaced <globals>
*/
int readGlobals() {

	int fd, i, flags;
	globals_t aglob;

	flags= O_RDONLY;
#ifdef WIN32
	flags|= O_BINARY;
#endif

	fd= open(globals.globalsName, flags);
	if ( fd== -1 ) return 0;
	else {
		i= read(fd,&aglob,sizeof(globals_t) );
		if ( i != sizeof(globals_t) ) {
			fprintf(stderr,"Short read (%d bytes, not %d) of %s, aborting\n",
				i, sizeof(globals_t), globals.globalsName ), 
			exit(1);
		}
		close(fd);
		globals= aglob;
		return 1;
	}
}

#if 0
/*##############################*/
/*--------------------------------------------------------------------
* Returns 1 if the globals file is successfully written.
* Otherwise, complain bitterly and return 0
* 
* Writes coord arrays at the end of the globals struct if
* <everything> is not zero.
*/

int writeGlobals(int everything) {
	int fd, i, flags;
    int nbytes;

	flags= O_RDWR | O_CREAT | O_TRUNC;
#ifdef WIN32
	flags|= O_BINARY;
#endif

	nbytes= sizeof(globals_t);
	/*
	if ( !everything )
		nbytes -= (sizeof(globals.xxl) + sizeof(globals.yyl) + sizeof(globals.zzl));
*/
	fd= open(globals.globalsName, flags, 0644 );
	if ( fd== -1 ) { perror(globals.globalsName); return 0; }
	else {
		i= write(fd,&globals, nbytes );
		if ( i != nbytes ) {
			perror("cannot write globals file!!!");
			return 0;
		}
		fdatasync(fd);
		close(fd);
		/* fprintf(stderr,"tm: wrote %d byte '%s' globals file\n",
			i,globals.globalsName); */
		return 1;
	}
}


int WRITEGLOBALS( int *everything ) { return writeGlobals(*everything); }

/*##############################*/
#endif


/*----------------------------------------------------------------------
* Dump brick of bytes  - debugging at the moment
*/

void WRITEBOB( char *prefix, int *ndump, int *nx, int *ny, int *nz, char *data ) {

	int fd, nbytes;
    char name[256];

	sprintf(name,"%s%04d", prefix, *ndump );

#ifdef WIN32
	fd= open(name, O_RDWR|O_CREAT|O_BINARY, 0644);
#else
	fd= open(name, O_RDWR|O_CREAT, 0644);
#endif
	if ( fd == -1 ) {
		fprintf(stderr,"Can't open '%s'\n", name);
		return;
	}
	nbytes= *nx * *ny * *nz;
	write(fd,data,nbytes);
	close(fd);
}

/*----------------------------------------------------------------
 */
static FILE *trace;

void OPENTRACE() {
  trace= fopen( "trace", "a" );
}

void CLOSETRACE() {
  fclose(trace);
}

void WRITETRACE( char *line, int len ) {
  int i;
  i= len-1;
  while( i>0 && ( line[i]==' ' || line[i]==0 ) ) --i;
  line[++i]= '\n';
  line[++i]= 0;
  fputs( line, trace );
  fflush( trace );
}

void SLEEP( int *nsec ) {
  Sleep(*nsec);
}

void MKDIR( char *name, int len ) {
  int ierr;
  char cname[256];
  strncpy(cname,name,len);
  cname[len]= 0;
  /* fprintf(stderr,"making directory '%s'\n", cname ); */
  ierr= mkdir( cname, 0755 );
  /*   if ( ierr && ierr!= EEXIST ) perror(cname);*/
}

/* Create empty file named 'xxxxdone' for file xfer daemon's benefit */

void CDDONE( char *dirname, int *dumpnum ) {
  char name[256], digit[20];
  int fd;

  strcpy(name,dirname);
  sprintf(digit,"done%04d",*dumpnum );
  strcat(name,digit);
  fd= open( name, O_WRONLY| O_CREAT, 0644 );
  if ( fd== -1 ) perror("dirname");
  else close(fd);
}

/* Append strip info to 3 FE files.  These binary files are named [x|y|z]strip-xyz where x=tilex,
 * y=tile z=tilez.  (here single digit, as is done for'thing' tile names.)  For example,
 * "x-strip-111".  The concatenation of these files is left as an exercise for the reader, dhp.
 * These files are placed in the same directory the compressed dumps are - rootDir.
 * Only one strip globally is used, and the one selected is in tiles which have a tile index
 * in x, y and/or z of '1'.  the 'origin' tile, 1,1,1 will have all 3 files written.
 *
 * The strip files contain records of the form: (say for an x-strip):
 * nstep  t  rho(1:npx)  prs(1:npx)  uux(1:npx)  uuy(1:npx)  uuz(1:npx)  diag(1:npx)
 * where nstep is an integer*4, the rest are floats.
 *
 */

void STRIPOUT( char *axis, int *npxyz,
			   int *itx, int *ity, int *itz, int *nstep, float *t,
			   float *rho, float *prs, float *uux, float *uuy, float *uuz, float *diag ) {
  int fd;
  char name[256];
  float *rp;
  float record[2+6*2048];  /* max tile size ridiculously large 2048 */
  int nbytes, nfloats, n;
  
  /* Compose name */
  sprintf(name,"%s%c-strip-%d%d%d", globals.homeDir, *axis, *itx, *ity, *itz );

  fd= open(name, O_RDWR|O_CREAT, 0644 );
  if ( fd==-1 ) { perror(name); exit(1); }
  lseek( fd, 0, SEEK_END );
  /* compose & write record.  Here I'm assuming the size of float==int for this quick&dirty
   * output.  Shrug.
   */
  memcpy(record,nstep,sizeof(int));
  memcpy(record+1, t, sizeof(float));
  nfloats= *npxyz;
  rp= record+2;
  nbytes= nfloats * sizeof(float);
  memcpy(rp, rho, nbytes );  rp += nfloats;
  memcpy(rp, prs, nbytes );  rp += nfloats;
  memcpy(rp, uux, nbytes );  rp += nfloats;
  memcpy(rp, uuy, nbytes );  rp += nfloats;
  memcpy(rp, uuz, nbytes );  rp += nfloats;
  memcpy(rp, diag, nbytes );  rp += nfloats;

  nbytes= (2 + 6*nfloats) * sizeof(float);
  n= write( fd, record, nbytes );
  if ( n != nbytes ) { perror("short write of strip file"); exit(1); }
  close(fd);
}

void coreme() {
#include <signal.h>
  signal(SIGSEGV,SIG_DFL);
}

void SETSTACK() { 
  int i;
  struct rlimit rlim;
  i= getrlimit( RLIMIT_STACK, &rlim );
  /*  fprintf(stderr," ierr=%d, stack limit= %d\n", i, rlim.rlim_cur );*/
  rlim.rlim_cur= 32*1024*1024;
  i= setrlimit( RLIMIT_STACK, &rlim );

  /*   fprintf(stderr," ierr=%d, stack limit= %d\n", i, rlim.rlim_cur );*/
  i= getrlimit( RLIMIT_STACK, &rlim );
  fprintf(stderr," ierr=%d, stack limit= %d\n", i, rlim.rlim_cur );
  coreme();
}