/*
 * Copyright 2020 NVIDIA Corporation.  All rights reserved.
 *
 * Please refer to the NVIDIA end user license agreement (EULA) associated
 * with this source code for terms and conditions that govern your use of
 * this software. Any use, reproduction, disclosure, or distribution of
 * this software and related documentation outside the terms of the EULA
 * is strictly prohibited.
 */

#include <stdlib.h>
#include <pthread.h>
#include <stdio.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>

// CUDA includes
#include <cuda.h>
#include <cuda_runtime.h>

#include "cufile.h"
#include "cufile_sample_utils.h"
/*
 * This sample shows how two threads independently open the file and
 * have a separate copy of CUfileHandle_t.
 */

#define TOGB(x) ((x)/(1024*1024*1024L))
#define GB(x) ((x)*1024*1024*1024L)
#define MB(x) ((x)*1024*1024L)
#define KB(x) ((x)*1024L)

static void* thread_fn(void *data)
{
	const int fd = *(int *)data;
	void *gpubuffer;
	CUfileError_t status;
	CUfileDescr_t cfr_descr;
	CUfileHandle_t cfr_handle;
	int i, nr_ios;
	loff_t offset = 0;
	

	memset((void *)&cfr_descr, 0, sizeof(CUfileDescr_t));
	cfr_descr.handle.fd = fd;
	cfr_descr.type = CU_FILE_HANDLE_TYPE_OPAQUE_FD;
	status = cuFileHandleRegister(&cfr_handle, &cfr_descr);
	if (status.err != CU_FILE_SUCCESS) {
		printf("file register error: %s\n", CUFILE_ERRSTR(status.err));
		close(fd);
	        exit(1);	
	}

	check_cudaruntimecall(cudaSetDevice(0));
	check_cudaruntimecall(cudaMalloc(&gpubuffer, MB(1)));

        /*
         * Each thread allocates GPU Memory
         * and invoke cuFileBufRegister on the entire memory. This
         * is done once; After registering the buffer, we read
         * the data in chunks to the same buffer. We are using the GPU Buffer
	 * as a streaming buffer wherein we populate the data to the same
	 * buffer. This is optimal as data is directly DMA'ed to the registered
	 * buffer.
         */
	status = cuFileBufRegister(gpubuffer, MB(1), 0);
	if (status.err != CU_FILE_SUCCESS) {
		printf("Buffer register failed :%s\n", CUFILE_ERRSTR(status.err));
		cudaFree(gpubuffer);
	        exit(1);	
	}

	nr_ios = GB(1)/MB(1);

	for (i = 0; i < nr_ios; i++) {
		int ret;

		ret = cuFileRead(cfr_handle, gpubuffer, MB(1), offset, 0);
		if (ret < 0) {
			fprintf(stderr, "cuFile Read failed with ret=%d\n", ret);
			goto err;
		}

		offset += MB(1);
	}

err:
	status = cuFileBufDeregister(gpubuffer);
        if (status.err != CU_FILE_SUCCESS) {
        	fprintf(stderr, "Buffer Deregister failed :%s\n", CUFILE_ERRSTR(status.err));
	}

	check_cudaruntimecall(cudaFree(gpubuffer));
	fprintf(stdout, "Read Success from fd %d to GPU id 0\n", fd);
	close(fd);
	return NULL;
}

void help(void) {
	printf("\n./cufilesample_009 <file-path-1> <file-path-2>\n");
}

int main(int argc, char **argv) {

	pthread_t thread1, thread2;

	if (argc < 3) {
		fprintf(stderr, "Invalid input.\n");
		help();
		exit(1);
	}

	int fd1  = open(argv[1], O_RDWR | O_DIRECT);

	if (fd1 < 0) {
		fprintf(stderr, "Unable to open file %s fd %d\n",
				argv[1], fd1);
		exit(1);
	}

	int fd2  = open(argv[2], O_RDWR | O_DIRECT);

	if (fd2 < 0) {
		fprintf(stderr, "Unable to open file %s fd %d\n",
				argv[2], fd2);
                close(fd1);
		exit(1);
	}

	pthread_create(&thread1, NULL, &thread_fn, (void *)&fd1);
	pthread_create(&thread2, NULL, &thread_fn, (void *)&fd2);

	pthread_join(thread1, NULL);
	pthread_join(thread2, NULL);

	return 0;
}
