[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[Crash-utility] [PATCH] crash: Support a dumpfile in the flattened format.



    
Hi Dave,
    
makedumpfile command can create a dumpfile in the flattened format by -F
option, the format is useful for transporting the dump data by SSH.
    
But the crash utility could not read the dumpfile directly, so a user
should create a readable dumpfile by -R option of makedumpfile.
That was not userfriendly, and this patch is the solution for it.
    
If applying this patch to the crash utility, it can read a dumpfile in
the flattened format directly.


Thanks,
Ken'ichi Ohmichi

---    
Signed-off-by: Ken'ichi Ohmichi <oomichi mxs nes nec co jp>

diff --git a/Makefile b/Makefile
index 5fb52d6..d5eac0a 100644
--- a/Makefile
+++ b/Makefile
@@ -78,7 +78,7 @@ INSTALLDIR=${DESTDIR}/usr/bin
 
 GENERIC_HFILES=defs.h xen_hyper_defs.h
 MCORE_HFILES=va_server.h vas_crash.h
-REDHAT_HFILES=netdump.h diskdump.h xendump.h kvmdump.h qemu-load.h
+REDHAT_HFILES=netdump.h diskdump.h makedumpfile.h xendump.h kvmdump.h qemu-load.h
 LKCD_DUMP_HFILES=lkcd_vmdump_v1.h lkcd_vmdump_v2_v3.h lkcd_dump_v5.h \
         lkcd_dump_v7.h lkcd_dump_v8.h
 LKCD_OBSOLETE_HFILES=lkcd_fix_mem.h
@@ -93,7 +93,7 @@ CFILES=main.c tools.c global_data.c memory.c filesys.c help.c task.c \
 	extensions.c remote.c va_server.c va_server_v1.c symbols.c cmdline.c \
 	lkcd_common.c lkcd_v1.c lkcd_v2_v3.c lkcd_v5.c lkcd_v7.c lkcd_v8.c\
 	lkcd_fix_mem.c s390_dump.c lkcd_x86_trace.c \
-	netdump.c diskdump.c xendump.c unwind.c unwind_decoder.c \
+	netdump.c diskdump.c makedumpfile.c xendump.c unwind.c unwind_decoder.c \
 	unwind_x86_32_64.c unwind_arm.c \
 	xen_hyper.c xen_hyper_command.c xen_hyper_global_data.c \
 	xen_hyper_dump_tables.c kvmdump.c qemu.c qemu-load.c
@@ -109,7 +109,7 @@ OBJECT_FILES=main.o tools.o global_data.o memory.o filesys.o help.o task.o \
 	arm.o \
 	extensions.o remote.o va_server.o va_server_v1.o symbols.o cmdline.o \
 	lkcd_common.o lkcd_v1.o lkcd_v2_v3.o lkcd_v5.o lkcd_v7.o lkcd_v8.o \
-	lkcd_fix_mem.o s390_dump.o netdump.o diskdump.o xendump.o \
+	lkcd_fix_mem.o s390_dump.o netdump.o diskdump.o makedumpfile.o xendump.o \
 	lkcd_x86_trace.o unwind_v1.o unwind_v2.o unwind_v3.o \
 	unwind_x86_32_64.o unwind_arm.o \
 	xen_hyper.o xen_hyper_command.o xen_hyper_global_data.o \
@@ -453,6 +453,9 @@ netdump_daemon.o: ${GENERIC_HFILES} ${REDHAT_HFILES} netdump.c
 diskdump.o: ${GENERIC_HFILES} ${REDHAT_HFILES} diskdump.c
 	cc -c ${CRASH_CFLAGS} diskdump.c ${WARNING_OPTIONS} ${WARNING_ERROR}
 
+makedumpfile.o: ${GENERIC_HFILES} ${REDHAT_HFILES} makedumpfile.c
+	cc -c ${CRASH_CFLAGS} makedumpfile.c ${WARNING_OPTIONS} ${WARNING_ERROR}
+
 xendump.o: ${GENERIC_HFILES} ${REDHAT_HFILES} xendump.c
 	cc -c ${CRASH_CFLAGS} xendump.c ${WARNING_OPTIONS} ${WARNING_ERROR}
 
diff --git a/defs.h b/defs.h
index 22f876b..27ee9cd 100755
--- a/defs.h
+++ b/defs.h
@@ -4559,6 +4559,12 @@ int dumpfile_is_split(void);
 void show_split_dumpfiles(void);
 
 /*
+ * makedumpfile.c
+ */
+void check_flattened_format(char *file);
+int read_dump_file(int fd, off_t offset, void *buf, size_t size);
+
+/*
  * xendump.c
  */
 int is_xendump(char *);
diff --git a/diskdump.c b/diskdump.c
index fca4a50..ebb3838 100644
--- a/diskdump.c
+++ b/diskdump.c
@@ -207,15 +207,8 @@ restart:
 	if ((header = realloc(header, block_size)) == NULL)
 		error(FATAL, "diskdump / compressed kdump: cannot malloc block_size buffer\n");
 
-	if (lseek(dd->dfd, 0, SEEK_SET) == failed) {
-		if (CRASHDEBUG(1))
-			error(INFO, "diskdump / compressed kdump: cannot lseek dump header\n");
-		goto err;
-	}
-
-	if (read(dd->dfd, header, block_size) < block_size) {
-		if (CRASHDEBUG(1))
-			error(INFO, "diskdump / compressed kdump: cannot read dump header\n");
+	if (!read_dump_file(dd->dfd, 0, header, block_size)) {
+		error(FATAL, "diskdump / compressed kdump: cannot read header\n");
 		goto err;
 	}
 
@@ -280,19 +273,12 @@ restart:
 
 	/* read sub header */
 	offset = (off_t)block_size;
-	if (lseek(dd->dfd, offset, SEEK_SET) == failed) {
-		error(INFO, "%s: cannot lseek dump sub header\n",
-			DISKDUMP_VALID() ? "diskdump" : "compressed kdump");
-
-		goto err;
-	}
 
 	if (DISKDUMP_VALID()) {
 		if ((sub_header = malloc(block_size)) == NULL)
 			error(FATAL, "diskdump: cannot malloc sub_header buffer\n");
 
-		if (read(dd->dfd, sub_header, block_size)
-		  < block_size) {
+		if (!read_dump_file(dd->dfd, offset, sub_header, block_size)) {
 			error(INFO, "diskdump: cannot read dump sub header\n");
 			goto err;
 		}
@@ -301,8 +287,7 @@ restart:
 		if ((sub_header_kdump = malloc(block_size)) == NULL)
 			error(FATAL, "compressed kdump: cannot malloc sub_header_kdump buffer\n");
 
-		if (read(dd->dfd, sub_header_kdump, block_size)
-		  < block_size) {
+		if (!read_dump_file(dd->dfd, offset, sub_header_kdump, block_size)) {
 			error(INFO, "compressed kdump: cannot read dump sub header\n");
 			goto err;
 		}
@@ -314,24 +299,18 @@ restart:
 	dd->bitmap_len = bitmap_len;
 
 	offset = (off_t)block_size * (1 + header->sub_hdr_size);
-	if (lseek(dd->dfd, offset, SEEK_SET) == failed) {
-		error(INFO, "%s: cannot lseek memory bitmap\n",
-			DISKDUMP_VALID() ? "diskdump" : "compressed kdump");
-
-		goto err;
-	}
 
 	if ((dd->bitmap = malloc(bitmap_len)) == NULL)
 		error(FATAL, "%s: cannot malloc bitmap buffer\n",
 			DISKDUMP_VALID() ? "diskdump" : "compressed kdump");
 
 	dd->dumpable_bitmap = calloc(bitmap_len, 1);
-	if (read(dd->dfd, dd->bitmap, bitmap_len) < bitmap_len) {
+
+	if (!read_dump_file(dd->dfd, offset, dd->bitmap, bitmap_len)) {
 		error(INFO, "%s: cannot read memory bitmap\n",
 			DISKDUMP_VALID() ? "diskdump" : "compressed kdump");
 		goto err;
 	}
-
 	if (dump_is_partial(header))
 		memcpy(dd->dumpable_bitmap, dd->bitmap + bitmap_len/2,
 		       bitmap_len/2);
@@ -370,17 +349,11 @@ restart:
 		size = sub_header_kdump->size_note;
 		offset = sub_header_kdump->offset_note;
 
-		if (lseek(dd->dfd, offset, SEEK_SET) == failed) {
-			error(INFO, "compressed kdump: cannot lseek dump elf"
-				" notes\n");
-			goto err;
-		}
-
 		if ((notes_buf = malloc(size)) == NULL)
 			error(FATAL, "compressed kdump: cannot malloc notes"
 				" buffer\n");
 
-		if (read(dd->dfd, notes_buf, size) < size) {
+		if (!read_dump_file(dd->dfd, offset, notes_buf, size)) {
 			error(INFO, "compressed kdump: cannot read notes data"
 				"\n");
 			goto err;
@@ -624,21 +597,17 @@ cache_page(physaddr_t paddr)
 	desc_pos = pfn_to_pos(pfn);
 	seek_offset = dd->data_offset
 			+ (off_t)(desc_pos - 1)*sizeof(page_desc_t);
-	lseek(dd->dfd, seek_offset, SEEK_SET);
 
 	/* read page descriptor */
-	if (read(dd->dfd, &pd, sizeof(pd)) != sizeof(pd))
+	if (!read_dump_file(dd->dfd, seek_offset, &pd, sizeof(pd)))
 		return READ_ERROR;
 
 	/* sanity check */
 	if (pd.size > block_size)
 		return READ_ERROR;
 
-	if (lseek(dd->dfd, pd.offset, SEEK_SET) == failed)
-		return SEEK_ERROR;
-
 	/* read page data */
-	if (read(dd->dfd, dd->compressed_page, pd.size) != pd.size)
+	if (!read_dump_file(dd->dfd, pd.offset, dd->compressed_page, pd.size))
 		return READ_ERROR;
 
 	if (pd.flags & DUMP_DH_COMPRESSED) {
@@ -833,17 +802,12 @@ static void dump_vmcoreinfo(FILE *fp)
 	off_t offset = dd->sub_header_kdump->offset_vmcoreinfo;
 	const off_t failed = (off_t)-1;
 
-	if (lseek(dd->dfd, offset, SEEK_SET) == failed) {
-		error(INFO, "compressed kdump: cannot lseek dump vmcoreinfo\n");
-		return;
-	}
-
 	if ((buf = malloc(size_vmcoreinfo)) == NULL) {
 		error(FATAL, "compressed kdump: cannot malloc vmcoreinfo"
 				" buffer\n");
 	}
 
-	if (read(dd->dfd, buf, size_vmcoreinfo) < size_vmcoreinfo) {
+	if (!read_dump_file(dd->dfd, offset, buf, size_vmcoreinfo)) {
 		error(INFO, "compressed kdump: cannot read vmcoreinfo data\n");
 		goto err;
 	}
diff --git a/main.c b/main.c
index 892eb4e..d65a13a 100755
--- a/main.c
+++ b/main.c
@@ -379,6 +379,8 @@ main(int argc, char **argv)
 
 		} else if (!(pc->flags & KERNEL_DEBUG_QUERY)) {
 
+			check_flattened_format(argv[optind]);
+
 			if (STREQ(argv[optind], "/dev/mem")) {
                         	if (pc->flags & MEMORY_SOURCES) {
                                 	error(INFO, 
diff --git a/makedumpfile.c b/makedumpfile.c
new file mode 100644
index 0000000..e1add9a
--- /dev/null
+++ b/makedumpfile.c
@@ -0,0 +1,297 @@
+/* 
+ * makedumpfile.c
+ * 
+ * This code is for reading a dumpfile ganarated by makedumpfile command.
+ *
+ * Copyright (C) 2011  NEC Soft, Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * Author: Ken'ichi Ohmichi <oomichi mxs nes nec co jp>
+ */
+
+#include "defs.h"
+#include "makedumpfile.h"
+#include <byteswap.h>
+
+int is_flattened_format = 0;
+
+struct flat_data {
+	int64_t off_flattened;
+	int64_t off_rearranged; /* offset which will be rearranged. */
+	int64_t buf_size;
+};
+
+struct all_flat_data {
+	unsigned long long	num_array;
+	struct flat_data	*array;
+	size_t			file_size;
+};
+
+struct all_flat_data afd;
+
+static int
+is_bigendian(void)
+{
+	int i = 0x12345678;
+
+	if (*(char *)&i == 0x12)
+		return TRUE;
+	else
+		return FALSE;
+}
+
+static unsigned long long
+store_flat_data_array(char *file, struct flat_data **fda)
+{
+	int			result = FALSE, fd;
+	int64_t			offset_fdh;
+	unsigned long long	num_allocated = 0;
+	unsigned long long	num_stored    = 0;
+	unsigned long long	size_allocated;
+	struct flat_data	*ptr = NULL, *cur;
+	struct makedumpfile_data_header	fdh;
+
+	fd = open(file, O_RDONLY);
+	if (fd < 0) {
+		error(INFO, "unable to open dump file %s", file);
+		return -1;
+	}
+	if (lseek(fd, MAX_SIZE_MDF_HEADER, SEEK_SET) < 0) {
+		error(INFO, "unable to seek dump file %s", file);
+		close(fd);
+		return -1;
+	}
+	while (1) {
+		if (num_allocated <= num_stored) {
+			num_allocated += 100;
+			size_allocated = sizeof(struct flat_data)
+					 * num_allocated;
+			ptr = realloc(ptr, size_allocated);
+			if (ptr == NULL) {
+				error(INFO, "unable to allocate");
+				break;
+			}
+		}
+		offset_fdh = lseek(fd, 0x0, SEEK_CUR);
+
+		if (read(fd, &fdh, sizeof(fdh)) < 0) {
+			error(INFO, "unable to read dump file %s", file);
+			break;
+		}
+		if (!is_bigendian()){
+			fdh.offset   = bswap_64(fdh.offset);
+			fdh.buf_size = bswap_64(fdh.buf_size);
+		}
+		if (fdh.offset == END_FLAG_FLAT_HEADER) {
+			result = TRUE;
+			break;
+		}
+		cur = ptr + num_stored;
+		cur->off_flattened  = offset_fdh + sizeof(fdh);
+		cur->off_rearranged = fdh.offset;
+		cur->buf_size       = fdh.buf_size;
+		num_stored++;
+
+		/* seek for next makedumpfile_data_header. */
+		if (lseek(fd, fdh.buf_size, SEEK_CUR) < 0) {
+			error(INFO, "unable to seek dump file %s", file);
+			break;
+		}
+	}
+	close(fd);
+	if (result == FALSE) {
+		free(ptr);
+		return -1;
+	}
+	*fda = ptr;
+
+	return num_stored;
+}
+
+static void
+sort_flat_data_array(struct flat_data **fda, unsigned long long num_fda)
+{
+	unsigned long long	i, j;
+	struct flat_data	tmp, *cur_i, *cur_j;
+
+	for (i = 0; i < num_fda - 1; i++) {
+		for (j = i + 1; j < num_fda; j++) {
+			cur_i = *fda + i;
+			cur_j = *fda + j;
+
+			if (cur_i->off_rearranged < cur_j->off_rearranged)
+				continue;
+
+			tmp.off_flattened  = cur_i->off_flattened;
+			tmp.off_rearranged = cur_i->off_rearranged;
+			tmp.buf_size       = cur_i->buf_size;
+
+			cur_i->off_flattened  = cur_j->off_flattened;
+			cur_i->off_rearranged = cur_j->off_rearranged;
+			cur_i->buf_size       = cur_j->buf_size;
+
+			cur_j->off_flattened  = tmp.off_flattened;
+			cur_j->off_rearranged = tmp.off_rearranged;
+			cur_j->buf_size       = tmp.buf_size;
+		}
+	}
+}
+
+static int
+read_all_makedumpfile_data_header(char *file)
+{
+	unsigned long long	num;
+	struct flat_data	*fda;
+
+	num = store_flat_data_array(file, &fda);
+	if (num < 0)
+		return FALSE;
+
+	sort_flat_data_array(&fda, num);
+
+	afd.num_array = num;
+	afd.array     = fda;
+
+	return TRUE;
+}
+
+void
+check_flattened_format(char *file)
+{
+	int fd;
+	struct makedumpfile_header fh;
+
+	fd = open(file, O_RDONLY);
+	if (fd < 0) {
+		error(INFO, "unable to open dump file %s", file);
+		return;
+	}
+	if (read(fd, &fh, sizeof(fh)) < 0) {
+		error(INFO, "unable to read dump file %s", file);
+		close(fd);
+		return;
+	}
+	close(fd);
+
+	if (!is_bigendian()){
+		fh.type    = bswap_64(fh.type);
+		fh.version = bswap_64(fh.version);
+	}
+	if ((strncmp(fh.signature, MAKEDUMPFILE_SIGNATURE, sizeof(MAKEDUMPFILE_SIGNATURE)) != 0) || 
+	    (fh.type != TYPE_FLAT_HEADER))
+		return;
+
+	if (!read_all_makedumpfile_data_header(file))
+		return;
+
+	is_flattened_format = TRUE;
+}
+
+static int
+read_raw_dump_file(int fd, off_t offset, void *buf, size_t size)
+{
+	if (lseek(fd, offset, SEEK_SET) < 0) {
+		if (CRASHDEBUG(1))
+			error(INFO, "cannot lseek dump file\n");
+		return FALSE;
+	}
+	if (read(fd, buf, size) < size) {
+		if (CRASHDEBUG(1))
+			error(INFO, "cannot read dump file\n");
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+static int
+read_flattened_dump_file(int fd, off_t offset, void *buf, size_t size)
+{
+	unsigned long long	index, index_start, index_end;
+	int64_t			range_start, range_end;
+	size_t			read_size, remain_size;
+	off_t			offset_read;
+	struct flat_data	*ptr;
+
+	index_start = 0;
+	index_end   = afd.num_array;
+
+	while (1) {
+		index = (index_start + index_end) / 2;
+		ptr = afd.array + index;
+		range_start = ptr->off_rearranged;
+		range_end   = ptr->off_rearranged + ptr->buf_size;
+
+		if ((range_start <= offset) && (offset < range_end)) {
+			/* Found a corresponding array. */
+			offset_read = (offset - range_start) + ptr->off_flattened;
+
+			if (offset + size < range_end) {
+				if (!read_raw_dump_file(fd, offset_read, buf, size))
+					return FALSE;
+				break;
+			}
+
+			/* Searh other array corresponding to remaining data. */
+			read_size   = range_end - offset;
+			remain_size = size - read_size;
+			if (!read_raw_dump_file(fd, offset_read, buf, read_size))
+				return FALSE;
+			if (!read_flattened_dump_file(fd, offset + read_size,
+					(char *)buf + read_size, remain_size))
+				return FALSE;
+			break;
+
+		} else if ((index == index_start) &&
+			   (index_start + 1 == index_end)) {
+			/*
+			 * Try to read not-written area. That is a common case,
+			 * because the area might be skipped by lseek().
+			 * This area should be the data filled with zero.
+			 */
+			ptr = afd.array + index_end;
+			if (offset + size < ptr->off_rearranged) {
+				memset(buf, 0x0, size);
+			} else {
+				read_size   = ptr->off_rearranged - offset;
+				remain_size = size - read_size;
+				memset(buf, 0x0, read_size);
+				if (!read_flattened_dump_file(fd,
+						offset + read_size,
+						(char *)buf + read_size,
+						remain_size))
+					return FALSE;
+			}
+			break;
+
+		} else if (offset < ptr->off_rearranged)
+			index_end   = index;
+		else
+			index_start = index;
+	}
+	return TRUE;
+}
+
+int
+read_dump_file(int fd, off_t offset, void *buf, size_t size)
+{
+	if (is_flattened_format) {
+		if (!read_flattened_dump_file(fd, offset, buf, size))
+			return FALSE;
+	} else {
+		if (!read_raw_dump_file(fd, offset, buf, size))
+			return FALSE;
+	}
+
+	return TRUE;
+}
+
diff --git a/makedumpfile.h b/makedumpfile.h
new file mode 100644
index 0000000..fdff4c4
--- /dev/null
+++ b/makedumpfile.h
@@ -0,0 +1,46 @@
+/* 
+ * makedumpfile.h
+ * 
+ * This code is for reading a dumpfile ganarated by makedumpfile command.
+ *
+ * Copyright (C) 2011  NEC Soft, Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * Author: Ken'ichi Ohmichi <oomichi mxs nes nec co jp>
+ */
+
+/*
+ * makedumpfile header
+ *   For re-arranging the dump data on different architecture, all the
+ *   variables are defined by 64bits. The size of signature is aligned
+ *   to 64bits, and change the values to big endian.
+ */
+#define MAKEDUMPFILE_SIGNATURE  "makedumpfile"
+#define NUM_SIG_MDF             (sizeof(MAKEDUMPFILE_SIGNATURE) - 1)
+#define SIZE_SIG_MDF            roundup(sizeof(char) * NUM_SIG_MDF, 8)
+#define SIG_LEN_MDF             (SIZE_SIG_MDF / sizeof(char))
+#define MAX_SIZE_MDF_HEADER     (4096) /* max size of makedumpfile_header */
+#define TYPE_FLAT_HEADER        (1)    /* type of flattened format */
+#define VERSION_FLAT_HEADER     (1)    /* current version of flattened format */
+#define END_FLAG_FLAT_HEADER    (-1)
+
+struct makedumpfile_header {
+	char    signature[SIG_LEN_MDF]; /* = "makedumpfile" */
+	int64_t type;
+	int64_t version;
+};
+
+struct makedumpfile_data_header {
+	int64_t offset;
+	int64_t buf_size;
+};
+
diff --git a/netdump.c b/netdump.c
index c6e7ad9..8244f51 100644
--- a/netdump.c
+++ b/netdump.c
@@ -121,11 +121,8 @@ is_netdump(char *file, ulong source_query)
 	}
 
 	size = MIN_NETDUMP_ELF_HEADER_SIZE;
-        if (read(fd, eheader, size) != size) {
-                sprintf(buf, "%s: read", file);
-                perror(buf);
+	if (!read_dump_file(fd, 0, eheader, size))
 		goto bailout;
-	}
 
         if (lseek(fd, 0, SEEK_SET) != 0) {
                 sprintf(buf, "%s: lseek", file);
@@ -290,12 +287,10 @@ is_netdump(char *file, ulong source_query)
 		clean_exit(1);
 	}
 
-        if (read(fd, tmp_elf_header, size) != size) {
-                sprintf(buf, "%s: read", file);
-                perror(buf);
+	if (!read_dump_file(fd, 0, tmp_elf_header, size)) {
 		free(tmp_elf_header);
-                goto bailout;
-        }
+		goto bailout;
+	}
 
 	nd->ndfd = fd;
 	nd->elf_header = tmp_elf_header;
@@ -393,12 +388,12 @@ file_elf_version(char *file)
 	}
 
 	size = MIN_NETDUMP_ELF_HEADER_SIZE;
-        if (read(fd, header, size) != size) {
-                sprintf(buf, "%s: read", file);
-                perror(buf);
+
+	if (!read_dump_file(fd, 0, header, size)) {
 		close(fd);
 		return -1;
 	}
+
 	close(fd);
 
 	elf32 = (Elf32_Ehdr *)&header[0];
@@ -523,11 +518,8 @@ read_netdump(int fd, void *bufptr, int cnt, ulong addr, physaddr_t paddr)
 		break;
 	}	
 
-        if (lseek(nd->ndfd, offset, SEEK_SET) == -1)
-                return SEEK_ERROR;
-
-        if (read(nd->ndfd, bufptr, cnt) != cnt)
-                return READ_ERROR;
+	if (!read_dump_file(nd->ndfd, offset, bufptr, cnt))
+		return READ_ERROR;
 
         return cnt;
 }


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]