/* threadhelper.c
 *
 * 7/23/2000 Sarah Anderson, LCSE
 * 2/20/2001 Merged win32/pthread+linux version - delete fortran binding
 * 3/28/2001 Added back fortran binding for t_create and abarrier and ifetchadd
 *
 * OS independent implementation of useful multithreading routines
 * including C and Fortran bindings.
 * 
 */

#ifdef WIN32
#include <windows.h>
#include <process.h>
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>  // assert turned off by defining NDEBUG for windows
#else
#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>
#include <sys/types.h>
#include <sys/ipc.h>
#include <sys/sem.h>
#include <assert.h>
#define DWORD int
#define T_LOCK t_lock_
#define T_UNLOCK t_unlock_
#define T_INIT t_init_
#endif

#define    MXTHREADS  33        /* Maximum no. compute threads         */
#define  MXFADDLOCKS  65
#define        debug   0

#include "threadhelper.h"

/*****************************************************************
* global data, all mutexes created with default attributes are fast
* ones, i.e. can be locked only once before being unlocked
*****************************************************************/

#define NSEM  10   /* one semaphore for ABARRIER(0), remainder for WAIT/RELEASE */
#define ABARRIERSEM 8  
#define NCRIT 8	  /* critical sections LOCK/UNLOCK */

/* interface to Linux/ia64 'fetch.c' atomic ops */
extern int fetch_and_add( int kounter, int increment );
extern void fetch_set( int kounter, int value );
extern int fetch_get( int kounter );

static unsigned long  threads  [ MXTHREADS   ];          /* array of thread handles */

#ifndef WIN32
pthread_attr_t   pthread_attributes;         /* attributes object for threads       */
pthread_mutex_t            lock_bar;         /* a lock  for fbarrier                */
pthread_mutex_t            lock_zap;         /* a lock  for fbarrier                */
pthread_mutex_t            lock_bars;        /* a lock  for abarrier                */
pthread_mutex_t            lock_fadd;        /*   locks for ifetchadd               */
pthread_mutex_t            lock_crit[NCRIT];    /*   lock for lock/unlock              */
static int Semid;
#include <semaphore.h>

#if 0
static sem_t Psem;  /* linux (posix) semaphore */
static sem_t Psem2;
#endif
#else

HANDLE Semid[NSEM];
static CRITICAL_SECTION Csection;
static CRITICAL_SECTION Critid[NCRIT];
#endif

static int Nthreads=0;              /* Number of spawned threads */
static unsigned int Stacksize= 0;

/*----------------------------------------------------------------
 */

long t_init( int node, unsigned int stacksize )
{
  int i;
  char name[80];
#ifndef WIN32
  int istatus;
  ushort zeros[NSEM];
  int semid_base;
#endif

  Stacksize  = stacksize;

#ifdef WIN32
  /* It seems NT is requiring me to name these */
  for(i=0; i<NSEM; ++i ) {
	sprintf(name,"sem%03d-%02d", node, i );
	//fprintf(stderr,"creating sem %d '%s'\n", i,name);
	Semid[i] = CreateSemaphore( NULL, 0, MXTHREADS, name );
  }

  for(i=0; i<NCRIT; ++i )
	InitializeCriticalSection( Critid+i );
#else

  for(i=0;i<NSEM;++i ) zeros[i]= 0;

  istatus= 0;
  istatus   |= pthread_mutex_init        (&lock_bar,  NULL);
  istatus   |= pthread_mutex_init        (&lock_zap,  NULL);
  istatus   |= pthread_mutex_init        (&lock_bars, NULL);
  istatus   |= pthread_mutex_init        (&lock_fadd, NULL);
  if ( istatus ) perror("pthread_mutex_init");

  /* zero counter(s) used by t_abarrier */
  fetch_set( 0,0 );
  fetch_set( 31,0 );

  /*   lock_fadd = PTHREAD_MUTEX_INITIALIZER; */

  istatus= 0;
  for (i=0; i<NCRIT; ++i ) 
	istatus   |= pthread_mutex_init        (&lock_crit[i], NULL);
  if ( istatus ) perror("pthread_mutex_init crit");

  /* Make semaphores for barrier functions (the 0th) and t_lock/t_release (1..) */

  semid_base = (('p'<<24) + ('p'<<16) + ('m'<<8)) + getpid();
  i=0;
  /* 10 attempts at unique ids, then bail */
  do { Semid = semget( semid_base + i, NSEM, IPC_CREAT|0600 ); }
  while(Semid == -1 && ++i < 10);

  if ( Semid < 0 ) { perror("ipcinit"); exit(1); }
  for(i=0; i<NSEM; ++i )
    if ( semctl( Semid, i, SETVAL, 0 ) ) { perror("ipcinit"); exit(1);}

#if 0
  if ( sem_init( &Psem, 0, 1 ) ) perror("sem_init");
  if ( sem_init( &Psem2, 0, 1 ) ) perror("sem_init2");
#endif

#endif

  return 0;
}

int T_INIT(int *n, int *stacksize) { return t_init(*n,*stacksize); }

#if 0
/* sleep milliseconds - debugging use only */
void usleep_(int *n) { usleep(*n * 10); }
void sleep_(int *n) { sleep(*n); }
#endif

/*================================================================
 * Remove semaphore set.
 */
void
t_shutdown() {
#ifdef WIN32
	int i;
	for (i=0; i<NSEM; ++i ) CloseHandle( Semid[i] );
#else
	semctl( Semid,0, IPC_RMID);
#endif
}

void t_shutdown_() { t_shutdown(); }

/*================================================================
 * Create thread for routine f_routine with thread handle ithread,
 * used by fortran calling routine to refer to this thread.  'ithread'
 * runs from 2..Nthreads, with 1 being the root thread.  
 *
 * Returns -1 on error, otherwise the 'handle' returned by the thread
 * creator.  
 */

long t_create( int *ithread, void *f_routine ) {
  void *ptr;
  long status;

#ifndef _WIN32
  pthread_t  threadId;
#endif

  ++Nthreads;
  threads[ Nthreads ] = Nthreads+1;

  /* thread is started with a single argument, the thread integer */
  ptr = (void*)ithread;
	    /* ???? (void*)(threads+Nthreads); */

#ifdef _WIN32
  status=  _beginthread( f_routine, 0, ptr );
  /* fprintf(stderr," beginthread returned %d\n", i ); */

#else

  status =  pthread_create(&threadId, NULL, f_routine, ptr );

#endif

  return status;
}

void create_( int *ithread, void *fun, int *ierr ) {
  *ierr= t_create( ithread, fun );
}


/*================================================================
 */

void t_lock(int n) {
  assert( n>=0 && n<NCRIT );
#ifdef WIN32
  EnterCriticalSection( &Critid[n] );
#else
  if ( pthread_mutex_lock(&lock_crit[n]) )  perror("pthread_mutex_lock");
#endif
}

void T_LOCK( int *n) { t_lock(*n); }

void t_unlock(int n) {
  assert( n>=0 && n<NCRIT );
#ifdef WIN32
  LeaveCriticalSection( &Critid[n] );
#else
  if ( pthread_mutex_unlock(&lock_crit[n]) ) perror("pthread_mutex_unlock");
#endif
}

void T_UNLOCK( int *n) { t_unlock(*n); }

#ifdef WIN32
/*****************************************************************
* Find id of calling thread
****************************************************************/

void t_id( int *ithread, int *istatus ) {

  DWORD myId;
  int i;

  myId= GetCurrentThreadId();
  for (i=0; i<Nthreads; ++i ) 
    if ( myId == threads[i] ) {
      *ithread = i+1;
      *istatus = 0;
      return;
    }
  *ithread= 1;  /* assume (root) thread 0 */
  *istatus= 1;  /* no match */
}
#endif

/*****************************************************************
 *This spinning barrier sychronization is a (p)thread implementation of
 *the sPPM routine fbarrier (spinning barrier)
 ****************************************************************/

void t_fbarrier( int num_threads )
{
  static volatile long bar = 0;
  static volatile long zap = 0; volatile long bars;
  int n = num_threads;
  int status= 0;
 
  if (n<=1) return;
  /* fprintf(stderr," fbarrier %d ", n ); */

  /* last thread in (n-1) sets zap to threadcount */

#ifdef WIN32
  bars = InterlockedExchangeAdd( (long*)&bar, 1 );
#else
    status  = pthread_mutex_lock   (&lock_bar);
    bars=bar; bar++;
    status += pthread_mutex_unlock (&lock_bar);
#endif

  if (bars == n-1) { bar=0; zap=n; }

  /* wait until last thread has set zap to n */
  while( !zap ) ;

  /* now each thread carefully backs out of the room.. */
#ifdef WIN32
  InterlockedExchangeAdd( (long*)&zap, -1 );
#else
  status += pthread_mutex_lock   (&lock_zap);
  zap--;
  status += pthread_mutex_unlock (&lock_zap);
#endif

  /* every thread has decremented zap */
  while( zap) ;
  /* fprintf(stderr,"\n"); */
  return;
}

void fbarrier_(unsigned *num) {
  t_fbarrier(*num);
}


/*----------------------------------------------------------------
 * Generalized atomic add.  Returns previous value.
 * Note this is NOT what sppm assumes.
 */
int t_ifetchadd( int kounter, int increment )
{

#ifdef WIN32
  return InterlockedExchangeAdd( &FetchKounter[k], increment );
#else
  int iw, ierr;

#if 0
  ierr= pthread_mutex_lock(&lock_fadd);
  if ( ierr ) perror("pthread_mutex_lock");
     *w= *w + increment;
  ierr= pthread_mutex_unlock(&lock_fadd);
  if ( ierr ) perror("pthread_mutex_unlock");
  return iw;
#else
  return fetch_and_add( kounter, increment );
#endif

#endif
}

int ifetchadd_( int *kounter, int *increment ) {
  return t_ifetchadd( *kounter, *increment);
}

void fetchset_( int *kounter, int *value ) {
  fetch_set( *kounter, *value );
}

#if 0
/* ****************************************************************
 *ifetchadd function for SPPM pthread version of SPPM routine
 * the 'lock_num' argument is historical, and unused... ?
 * (sppm style)
 */

int t_ifetchadd( int *w, int *num_lock )
{
  return InterlockedExchangeAdd( w, 1 );
}
#endif


static volatile u_long bars       = 0;

/*----------------------------------------------------------------
 */

void t_abarrier(unsigned  int num)
{
    struct sembuf sb;
    volatile int bars_local = 0;
    volatile int zap=0;
    int nm1, i;
    int status;

    nm1 = num - 1;
    if ( nm1<=0 ) return;
    sb.sem_flg = 0;
    sb.sem_num = 0;

#if 1
    /* Use reserved counter 0, not bars */
    bars_local= t_ifetchadd( 0, 1 );

    /*
    status   = pthread_mutex_lock     (&lock_bars);
    if ( status ) perror("pthread_mutex_lock");
    bars_local = bars++;
    status   = pthread_mutex_unlock   (&lock_bars);
    if ( status ) perror("pthread_mutex_unlock");
    */

#else
    sem_wait(&Psem2);
    bars_local = bars++;
    sem_post(&Psem2);
#endif

    if ( bars_local == nm1 )
    {
      /* for(i=nm1; i>0; --i) sem_post(&Psem); */
      fetch_set( 0,0 );
      fetch_set(31,num);
      /* bars = 0;*/
        sb.sem_op = nm1;   /*...notify */
    } else
    {
      /* sem_wait(&Psem); */
      sb.sem_op = -1;   /*... wait */
    }

    if ( semop( Semid, &sb, 1 ) ) perror("swait");

    while (! fetch_get(31) ) ;   /* wait for last thread to set to num */
    fetch_and_add( 31,-1 );
    while (fetch_get(31)) ;      /* wait for all threads to have decr. */
}

void abarrier_(unsigned *num) {
  t_abarrier(*num);
}


#if 0
/*----------------------------------------------------------------
 * A barrier which does not depend on pthread mutexs.
 * mythread is 1..num
 */

void t_bbarrier(unsigned mythread, unsigned  int nthreads)
{
  static volatile int btable[4]= {0,0,0,0};
  int i,n;

  btable[mythread-1]= 1;
  do for(n=i=0; i<nthreads; ++i ) n+= btable[i];
  while (n<nthreads);

  btable[mythread-1]= 0;
  do for(n=i=0; i<nthreads; ++i ) n+= btable[i];
  while (n);

}

void bbarrier_(int *mythread,unsigned *num) {
  t_bbarrier(*mythread,*num);
}

#endif


#if 0
/*===================*/

/*****************************************************************
*This is a sleeping barrier sychronization using semaphores pthread
*version of SPPM routine abarrier (sleeping barrier)
*****************************************************************/

void t_abarrier(unsigned int num)
{
    static volatile long bars       = 0;
           volatile long bars_local = 0;
    int nm1;

    nm1 = num - 1;
    if ( !nm1 ) return;

#ifdef WIN32
    bars_local = InterlockedExchangeAdd( (long*)&bars, 1 );
#else
    pthread_mutex_lock( &lock_bar );
    bars_local= bars; bars++;
    pthread_mutex_unlock (&lock_bar);
#endif
	
    if ( bars_local == nm1 )
    {
      //fprintf(stderr,"releasing.................\n");
        bars = 0;
	t_release(ABARRIERSEM);
	} else
    {
      //fprintf(stderr,"waiting..............\n");
      t_wait(ABARRIERSEM);
      /* wait */
    }
}

void abarrier_( unsigned *num ) { t_abarrier( *num ); }

/*===================*/
#endif


/* ****************************************************************
 * Semaphored Wait for release.  semnumber 0..NSEM-1, 0 reserved
 * ***************************************************************/

#ifdef WIN32

void t_wait( int n ) {
  assert( n>0 && n < NSEM );
  WaitForSingleObject( Semid[n], (DWORD)INFINITE );
}

#else
void t_wait( int n ) {
  int ierr;
  struct sembuf sb;
  assert( n>0 && n < NSEM );
  /*fprintf(stderr,"##### t_wait %d\n", n ); */
    sb.sem_num= n;
	sb.sem_op = -1;
	sb.sem_flg= 0;
	ierr= semop( Semid, &sb, 1 );
	if ( ierr ) perror("t_wait");
}
#endif

/* ****************************************************************
 * Release threads.  semnumber 0..NSEM-1, 0 reserved for barrier fun
 * ***************************************************************/

#ifdef WIN32
void t_release( int n ) {
  assert( n>0 && n < NSEM );
  ReleaseSemaphore( Semid[n], 1, NULL );
}

#else
void t_release( int n ) {
  int ierr;
  struct sembuf sb;
  assert( n>0 && n < NSEM );
  /*fprintf(stderr,"##### t_release %d\n", n ); */
    sb.sem_num= n;
	sb.sem_op = 1;
	sb.sem_flg= 0;
	ierr= semop( Semid, &sb, 1 );
	if ( ierr ) perror("t_release");
}
#endif