Re: where does the memory go?

["M. Nicklaus" <mn1 helix nih gov>]

On Tue, 9 Mar 1999, Daniel J. Frasnelli wrote:

>       Hmmmmmm. I bet your performance dropped like a rock after running
> that.  You realize that the kernel will throw out cached data only 
> when it needs to put more data in, correct? This is what keeps the
> "free" count low. 

In fact, if you tell fragbuster to allocate all of physical memory, the
kernel will swap out your application(s) entirely, which will reduce their
speed to zero.  But this slowdown lasts essentially only for the duration
of the fragbuster run, typically <2 min (just clocked 1:44 on a 512 MB 
500 MHz machine), and the applications recover very quickly thereafter as
their pages are loaded back into memory.

I can live very happily with this side effect if it gives me the benefit
of not having to, e.g., reboot the system.  Not being able to telnet into
it any more is not just "alarming", it may prevent me from using the
system altogether in the way I want to.  In particular for our
"number-crunching" runs, which may take up days or even weeks of CPU time,
losing a few minutes is totally insignificant.  But the situation may of
course be different if you're running a Web server, or a database server
etc.  So YMMV.


On Wed, 10 Mar 1999, Greg Lindahl wrote:

> I could imagine that this could be due to a lack of zeroed pages. But
> your fix is pretty drastic for a problem that I've never seen. So this
> bug has to be fairly obscure.
>
> > Often, it seems that network-related processes contribute to this
> > behavior, especially if there's a problem with the network, or some faulty
> > network setup etc.  We've had cases where a 512 MB machine came up with
> > 4 kB free memory after RPC'ing like crazy during boot-up.
>
> So? That doesn't mean anything. The only symptom you've described
> that's alarming is that you can't start some processes. Having low
> "free" memory is a feature of Linux disk buffering, not a bug.

See above.  Let me just add: If you have never experienced any ill effect
from this bug/feature/whatever, count yourself lucky, and simply ignore
this whole issue.  I'm not even claiming that our interpretation of the
problem is correct.  But our little program has repeatedly bailed us out,
and that's good enough for me.

For those who asked for fragbuster, the C code is appended below.
No guarantees of any kind are made, so use it at your own risk.
We've also never tested what happens if you tell it to allocate more
than physical memory + swap space combined -- so, in every respect,
caveat emptor.

Marc


-------------------------- fragbuster.c --------------------------
#include <stdio.h>

#include <stdlib.h>
#include <assert.h>

#define BOOL int
#define LONG long

#define KILO (double)1024 
#define MEGA (KILO * KILO)
#define GIGA (KILO * MEGA)
double memory_size = 0 ;
double Chunk = 10 * MEGA ;
double Max_memory = 128 * MEGA;

char *p = NULL ;



double bytes_to_mega( double mem) ;
double mega_to_bytes( double mem) ;
double find_largest_block( double max_size_bytes, double  chunk);
double progressive_alloc_blocks( double max_size_bytes, double  chunk, BOOL write_flag);

void main(int argc, char *argv[])
{

        double start_memory = -1 ;
        double end_memory = - 1 ;
        
        assert( sizeof( LONG) >=4) ;
        assert( (size_t)Max_memory > 0) ;
        assert( argc > 0) ;
        
        assert( argv[0] ) ;
	printf("---------------------------------------------------------------------\n");
        printf("Usage: %s [Max_memoryMb]\n",argv[0] );
        printf("Max_memoryMb should be the size of the physical memory. The default value is %.0f Mb\n", bytes_to_mega(Max_memory));
	printf("---------------------------------------------------------------------\n");
        if( argc == 2 )
        {
                assert( argv[1] ) ;
                if( (sscanf( argv[1], "%lf",&Max_memory) != 1) || (Max_memory < bytes_to_mega(Chunk)) )
                {
                        printf(" Invalid max memory parameter. Must be a number >= %g. Exiting\n", bytes_to_mega(Chunk)) ;
                        exit(EXIT_FAILURE) ;
                }
                Max_memory = mega_to_bytes(Max_memory) ;
        }
                        

        
        start_memory = find_largest_block(Max_memory, Chunk);
	printf("---------------------------------------------------------------------\n");
        
        progressive_alloc_blocks(  Max_memory, Chunk, 0) ;
	printf("---------------------------------------------------------------------\n");
        
        find_largest_block(Max_memory, Chunk);
	printf("---------------------------------------------------------------------\n");
        
        progressive_alloc_blocks(  Max_memory, Chunk, 1) ;
	printf("---------------------------------------------------------------------\n");
        
        
        end_memory = find_largest_block(Max_memory, Chunk);
        
	printf("---------------------------------------------------------------------\n");
        printf("Results summary:\n");
        printf(" Difference between up and down allocations:\n\
%.0f - %.0f => %.0f (should be 0 in absence of memory fragmentation)\n", 
                 bytes_to_mega(end_memory), bytes_to_mega(start_memory), bytes_to_mega(end_memory - start_memory));
	printf("---------------------------------------------------------------------\n");
        
        
        exit( EXIT_SUCCESS) ;           

}


/******************************************************************/
double find_largest_block( double max_size_bytes, double  chunk)
{
        double memory = 0 ;
        
        assert( max_size_bytes > 0) ;
        printf( "Trying to allocate the largest memory block starting from %.0f Mb by %.0f Mb decrements\n",
                        bytes_to_mega( max_size_bytes), bytes_to_mega( chunk)) ;
        
        
        for( memory = max_size_bytes ; memory > 0 ; memory -= Chunk )
        {
                char * p = malloc( memory ) ;
                if( p )
                {
                        free( p) ; p = NULL ;
                        break ;
                }
        }                       
        
        assert( memory <= max_size_bytes) ;     
        printf( "The largest memory block is %.0f Mb.\n", bytes_to_mega( memory)) ;
        
        return memory ;

}

/******************************************************************/
double progressive_alloc_blocks( double max_size_bytes, double  chunk, BOOL write_flag)
{
        double memory = 0 ;
        
        assert( max_size_bytes > 0) ;
        assert( chunk > 0) ;
        
        printf( "Progressive allocation of the memory block starting from %.0f Mb to %.0f Mb step:%.0f Mb \n",
                        bytes_to_mega(chunk), bytes_to_mega( max_size_bytes), bytes_to_mega(chunk) ) ;
        
        if( write_flag )
                printf( "The memory blocks will be written\n");
        else
                printf( "Allocation will be performed without real memory access\n");
        
                
        for( memory = chunk ; memory <= max_size_bytes ; memory += chunk )
        {
                char * p = malloc( memory ) ;
                if( p )
                {
                        printf( "%.0f Mb", bytes_to_mega( memory)) ;
                        assert( memory <= max_size_bytes) ;
                        if( write_flag )
                        {
                                LONG i = 0 ;
                                
                                for( i = 0 ; i < memory ; i += 1000 )
                                {
                                        p[i] = 'm' ;
                                }
                                printf( " W ") ;
                        }

                        printf( "\n") ;
                        
                        free( p) ;

                } else
                        break ;

        }                       
        
        printf( "\n") ;
        
        return memory ;

}
/******************************************************************/
double bytes_to_mega( double mem)
{
        return mem /MEGA ;
}

/******************************************************************/
double mega_to_bytes( double mem)
{
        return mem * MEGA ;
}