/* threadHelper.c
 *
 * 7/23/2000 Sarah Anderson, LCSE
 *
 * OS independent implementation of useful multithreading routines
 * including C and Fortran bindings.
 * 
 * Fortran binding:
 *
 * T_INIT( int *node, int *stacksize, int *status )
 *	Initialize structures
 * T_CREATE( int *ithread, void *f_routine, int *istatus )
 *	Spawn thread at f_routine.  *ithread counts 1.. and is the number passed
 *      to f_routine(ithread)
 * T_LOCK()
 *	Obtain simple lock
 * T_UNLOCK()
 *	Release simple lock
 * T_WAIT()
 *	Simple OS semaphore
 * T_RELEASE()
 *	Release thread(s) waiting on T_WAIT
 * T_ID( int *ithread, int *istatus )
 *	Partial implementation!
 * FBARRIER( int *num_threads, int *status )
 *	Spinning barrier
 * IFETCHADD( long *v, int *lock_number )
 *	Atomic increment of v (lock_number ignored)
 * ABARRIER( unsigned *num )
 *	Semaphore based barrier routine
 * LOOOPEND( volatile int *ivar )
 *	spins while *ivar != 0
 * IWAITER( volatile int *ivar )
 *	wpins while *ivar == 0
 */


#include <windows.h>
#include <process.h>
#include <assert.h>  // assert turned off by defining NDEBUG

#include <stdio.h>
#include <stdlib.h>

#define    MXTHREADS  33        /* Maximum no. compute threads         */
#define  MXFADDLOCKS  65
#define        debug   0

#include "threadhelper.h"

#ifdef _WIN32
/* For Digital/Compaq fortran: add __stdcall */
#define T_INIT T_INIT
#define T_CREATE  T_CREATE
#define T_ID  T_ID
#define SEM_DELETE  SEM_DELETE
#define FBARRIER  FBARRIER
#define ABARRIER  ABARRIER
#define IFETCHADD  IFETCHADD
#define IWAITER  IWAITER
#define LOOPEND  LOOPEND
#define T_RELEASE  T_RELEASE
#define T_WAIT T_WAIT
#endif

/*****************************************************************
* global data, all mutexes created with default attributes are fast
* ones, i.e. can be locked only once before being unlocked
*****************************************************************/

#define NSEM  9   /* one semaphore for ABARRIER, remainder for WAIT/RELEASE */
#define NCRIT 8	  /* critical sections LOCK/UNLOCK */

static unsigned long  threads  [ MXTHREADS   ];          /* array of thread handles */
static HANDLE Semid[NSEM];
static CRITICAL_SECTION Critid[NCRIT];

static int Nthreads=0;              /* Number of spawned threads */

static unsigned int Stacksize= 0;

//static CRITICAL_SECTION Csection;

/*****************************************************************
* global data, all mutexes created with default attributes are fast
* ones, i.e. can be locked only once before being unlocked
*****************************************************************/

long t_init( int node, unsigned int stacksize )
{
  int i;
  char name[80];

  Stacksize  = stacksize;
  /* fprintf(stdout,"init set stacksize to %d\n", Stacksize ); */

  /* It seems NT is requiring me to name these */
 
  for(i=0; i<NSEM; ++i ) {
	sprintf(name,"sem%03d-%02d", node, i );
	//fprintf(stderr,"creating sem %d '%s'\n", i,name);
	Semid[i] = CreateSemaphore( NULL, 0, MXTHREADS, name );
  }

  for(i=0; i<NCRIT; ++i )
	InitializeCriticalSection( Critid+i );
  // InitializeCriticalSection( &Csection );

  /* need error check on the above.. */
  return 0;
}

void T_INIT( int *node, unsigned int *stacksize, int *istatus ) {
  *istatus= t_init( *node, *stacksize ); 
}

t_shutdown() {
	int i;
	for (i=0; i<NSEM; ++i ) CloseHandle( Semid[i] );
}

/*****************************************************************
 * Create thread for routine f_routine with thread handle ithread,
 * used by fortran calling routine to refer to this thread.  'ithread'
 * runs from 2..Nthreads, with 1 being the root thread.  
 *
 * Returns -1 on error, otherwise the 'handle' returned by the thread
 * creator.  
 */

long t_create( int *ithread, void *f_routine ) {

  /* extern void CONTROL( int *ithread ); */
  long status;

  /* fprintf(stdout," creating thread ithread=%d\n", *ithread ); */
  if ( ithread != NULL ) {
	++Nthreads;
	threads[*ithread] = -1;
  }
  status=  _beginthread( f_routine, 0, ithread );
  
  /* fprintf(stderr," beginthread returned %d\n", i ); */

#if 0
  /* Set thread affinity - a frill perhaps */
  err= SetThreadIdealProcessor(i, *ithread - 1 );
  if ( err == -1 ) perror("setidealprocesor");
#endif

  return status;
}

void T_CREATE( int *ithread, void *f_routine, int *istatus ) {
  *istatus= t_create( ithread, f_routine );
}

/* ****************************************************************
 */

void T_LOCK(int *n) {
  assert( *n>=0 && *n<NCRIT ); 
  EnterCriticalSection( &Critid[*n] );
}
void t_lock(int n) {
  assert( n>=0 && n<NCRIT );
  EnterCriticalSection( &Critid[n] );
}

void T_UNLOCK(int *n) {
  assert( *n>=0 && *n<NCRIT );
	LeaveCriticalSection( &Critid[*n] );
}
void t_unlock(int n) {
  assert( n>=0 && n<NCRIT );
	LeaveCriticalSection( &Critid[n] );
}

/*****************************************************************
* Find id of calling thread
****************************************************************/

void t_id( int *ithread, int *istatus ) {

  DWORD myId;
  int i;

  myId= GetCurrentThreadId();
  for (i=0; i<Nthreads; ++i ) 
    if ( myId == threads[i] ) {
      *ithread = i+1;
      *istatus = 0;
      return;
    }
  *ithread= 1;  /* assume (root) thread 0 */
  *istatus= 1;  /* no match */
}

void T_ID( int *ithread, int *istatus ) {
  t_id( ithread, istatus );
}

/*****************************************************************
 *This spinning barrier sychronization is a (p)thread implementation of
 *the sPPM routine fbarrier (spinning barrier)
 ****************************************************************/

void t_fbarrier( int num_threads )
{
  static volatile long bar = 0;
  static volatile long zap = 0; volatile long bars;
  int n = num_threads;
 
  if (n<=1) return;
  /* fprintf(stderr," fbarrier %d ", n ); */

  /* last thread in (n-1) sets zap to threadcount */
  bars = InterlockedExchangeAdd( (long*)&bar, 1 );
  if (bars == n-1) { bar=0; zap=n; }

  /* wait until last thread has set zap to n */
  while( !zap ) ;

  /* now each thread carefully backs out of the room.. */
  InterlockedExchangeAdd( (long*)&zap, -1 );

  /* every thread has decremented zap */
  while( zap) ;
  /* fprintf(stderr,"\n"); */
  return;
}

void FBARRIER( int *num_threads )
{
  t_fbarrier( *num_threads );
}

/**************************************************************/
/* Delete semaphores for SPPM                                 */
/**************************************************************/

void t_sem_delete()
{
  int i;
  for (i=0; i<NSEM; ++i )
    CloseHandle( Semid[i] );
}
void SEM_DELETE() {
  t_sem_delete();
}

/*----------------------------------------------------------------
 * Generalized atomic add.  Returns previous value.
 * Note this is NOT what sppm assumes.
 */
int t_ifetchadd( int *w, int increment )
{
  return InterlockedExchangeAdd( w, increment );
}

int IFETCHADD ( int *w, int *increment )
{
  return InterlockedExchangeAdd( w, *increment );
}


#if 0
/* ****************************************************************
 *ifetchadd function for SPPM pthread version of SPPM routine
 * the 'lock_num' argument is historical, and unused... ?
 */

int t_ifetchadd( int *w, int *num_lock )
{
  return InterlockedExchangeAdd( w, 1 );
}

int IFETCHADD ( long *w, int *num_lock )
{
  return InterlockedExchangeAdd( w, 1 );
}
#endif

/*****************************************************************
*This is a sleeping barrier sychronization using semaphores pthread
*version of SPPM routine abarrier (sleeping barrier)
*****************************************************************/

void t_abarrier(unsigned int num)
{
    static volatile long bars       = 0;
           volatile long bars_local = 0;
    int nm1;

    nm1 = num - 1;
    if ( !nm1 ) return;

    bars_local = InterlockedExchangeAdd( (long*)&bars, 1 );
    if ( bars_local == nm1 )
    {
        bars = 0;
	    ReleaseSemaphore( Semid[0], 1, NULL );
        /* sb.sem_op = nm1;   notify */
	} else
    {
        WaitForSingleObject( Semid[0], (DWORD)INFINITE );
        /* wait */
    }
}

void ABARRIER (unsigned *num) {
  t_abarrier(*num);
}


/* ****************************************************************
 * Wait for release
 * ***************************************************************/

void T_WAIT( int *n ) {
  assert( *n > 0 && *n < NSEM );
  WaitForSingleObject( Semid[*n], (DWORD)INFINITE );
}
void t_wait( int n ) {
  assert( n>0 && n < NSEM );
  WaitForSingleObject( Semid[n], (DWORD)INFINITE );
}


/* ****************************************************************
 * Release threads
 * ***************************************************************/

void T_RELEASE( int *n ) {
  assert( *n > 0 && *n < NSEM );
  ReleaseSemaphore( Semid[*n], 1, NULL );
}
void t_release( int n ) {
  assert( n>0 && n < NSEM );
  ReleaseSemaphore( Semid[n], 1, NULL );
}


/*****************************************************************
*Twiddle (spin) until ivar changes.  volatile insures ivar fetched
*from memory for every while check
*****************************************************************/

void LOOPEND( volatile int *ivar )
{
        while(*ivar!=0);        /* spin */
}

void IWAITER( volatile int *ivar )
{
        while(*ivar==0);        /* spin */
}