[Cluster-devel] [PATCH] dlm: fix basts for granted PR waiting CW

[David Teigland]

On Wed, Jul 25, 2007 at 11:01:05AM -0500, David Teigland wrote:
> Fix a long standing bug where a blocking callback would be missed
> when there's a granted lock in PR mode and waiting locks in both
> PR and CW modes (and the PR lock was added to the waiting queue
> before the CW lock).  The logic simply compared the numerical values
> of the modes to determine if a blocking callback was required, but in
> the one case of PR and CW, the lower valued CW mode blocks the higher
> valued PR mode.  We just need to add a special check for this PR/CW
> case in the tests that decide when a blocking callback is needed.

Attached is a test I used in the past to trigger this bug (and others as
well).  I'd run two instances of the test on each of four nodes, like
this:

node01: rand_direct -d4 -f10     rand_direct -d8 -f10
node02: rand_direct -d4 -f10 -u  rand_direct -d8 -f10
node03: rand_direct -d4 -f10     rand_direct -d8 -f10
node04: rand_direct -d4 -f10     rand_direct -d8 -f10

Dave

-------------- next part --------------
/* tests to run:

   on two nodes: prealloc file, rand_direct -d1 -f1

   on two nodes: prealloc file, rand_direct -d1 -f1 -b

*/

#define _GNU_SOURCE
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <string.h>
#include <errno.h>
#include <ctype.h>
#include <limits.h>
#include <time.h>
#include <sys/stat.h>
#include <sys/file.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/times.h>
#include <sys/uio.h>
#include <sys/time.h>
#include <fcntl.h>

#define ONEMB 1048576

char *prog_name;
int quiet = 0;
int verbose = 0;
int delay = 0;
int enable_seeks = 0;
int enable_unlinks = 0;
int enable_buffered = 0;
int iterations = 0; /* forever */
int num_dirs = 10;
int num_files = 100;
int iobuf_size = ONEMB;
char *iobuf, **p_iobuf;

#define die(fmt, args...) \
{ \
  fprintf(stderr, "%s: ", prog_name); \
  fprintf(stderr, fmt, ##args); \
  exit(EXIT_FAILURE); \
}

void usage(void)
{
	printf("Usage:\n");
	printf("\n");
	printf("%s [options]\n", prog_name);
	printf("\n");
	printf("Options:\n");
	printf("\n");
	printf("  -b		random mix of buffered and direct io\n");
	printf("  -s		enable seeks before reads and writes\n");
	printf("  -u		enable unlinks\n");
	printf("  -i <n>	iterations, default 0 (forever)\n");
	printf("  -d <n>	number of dirs, default 10\n");
	printf("  -f <n>	number of files per dir, default 100\n");
}

int rand_int(int a, int b)
{
	return a + (int) (((float)(b - a + 1)) * random() / (RAND_MAX+1.0)); 
}

int do_read(int fd)
{
	read(fd, iobuf, iobuf_size);
	return 0;
}

int do_write(int fd)
{
	int rv;

	memset(iobuf, 0x55, iobuf_size);

	rv = write(fd, iobuf, iobuf_size);
	if (rv != iobuf_size)
		die("write size %d vs %d\n", rv, iobuf_size);
	return 0;
}

int do_seek(int fd)
{
	int off, rv;

	off = rand_int(0, ONEMB*100);
	off -= (off % 4096);
	
	rv = lseek(fd, off, SEEK_SET);

	return 0;
}

int rand_dio(void)
{
	if (rand_int(0, 1))
		return O_DIRECT;
	return 0;
}

int main(int argc, char *argv[])
{
	int i, j, flags, rv, c, fd, op, dir, file, ops = 0;
	char path[64];

	srandom(time(NULL));
	prog_name = argv[0];

	while ((c = getopt(argc, argv, "bqvsui:d:f:y:")) != -1) {
                switch (c) {
			case 'q':
				quiet = 1;
				break;
			case 'v':
				verbose = 1;
				break;
			case 'b':
				enable_buffered = 1;
				break;
			case 's':
				enable_seeks = 1;
				break;
			case 'u':
				enable_unlinks = 1;
				break;
			case 'i':
				iterations = atoi(optarg);
				break;
			case 'd':
				num_dirs = atoi(optarg);
				break;
			case 'f':
				num_files = atoi(optarg);
				break;
			case 'y':
				delay = atoi(optarg);
				break;
			case 'h':
			default:
				usage();
				exit(2);
		}
	}

	for (i = 0; i < num_dirs; i++) {
		memset(path, 0, sizeof(path));
		sprintf(path, "dir%.10u", i);
		mkdir(path, 0755);
	}

	p_iobuf = &iobuf;

	rv = posix_memalign((void *)p_iobuf, ONEMB, iobuf_size);
	if (rv) {
		perror("memalign error");
		exit(-2);
	}

	while (1) {
		op = rand_int(0, 5);
		dir = rand_int(0, num_dirs-1);
		file = rand_int(0, num_files-1);

		memset(path, 0, sizeof(path));
		sprintf(path, "dir%.10u/file%.10u", dir, file);

		if (verbose)
			printf("%s %d\n", path, op);

		if (op == 0) {
			/* open (write, append, create)
			   close */

			fd = open(path, O_WRONLY|O_CREAT|O_APPEND|O_DIRECT, 0644);
			if (fd < 0)
				die("open %s error %d errno %d\n",
				    path, fd, errno);
			close(fd);
			ops++;

		} else if (op == 1 || op == 2) {
			/* open (read, write)
			   write
			   close */

			if (enable_buffered)
				flags = rand_dio();
			else
				flags = O_DIRECT;

			fd = open(path, O_RDWR | flags);
			if (fd > 0) {
				if (enable_seeks)
					do_seek(fd);
				do_write(fd);
				close(fd);
				ops++;
			}

		} else if (op == 3 || op == 4) {
			/* open (read)
			   read
			   close */

			if (enable_buffered)
				flags = rand_dio();
			else
				flags = O_DIRECT;

			fd = open(path, O_RDONLY | flags);
			if (fd > 0) {
				if (enable_seeks)
					do_seek(fd);
				do_read(fd);
				close(fd);
				ops++;
			}

		} else if (op == 5) {
			if (enable_unlinks) {
				unlink(path);
				ops++;
			}

		} else {
			exit(-1);
		}

		if (!quiet)
			printf("%u:%d\n", ops, op);

		if (delay)
			sleep(delay);

		if (iterations && ops == iterations)
			break;
	}

	exit(EXIT_SUCCESS);
}