Threaded RT-application with memory locking and stack handling example

From RTwiki
(Difference between revisions)
Jump to: navigation, search
m
Line 1: Line 1:
 
====Threaded RT-application with memory locking and stack touching example====
 
====Threaded RT-application with memory locking and stack touching example====
  
 
   
 
 
     // Compile with 'gcc thisfile.c -lrt -Wall'
 
     // Compile with 'gcc thisfile.c -lrt -Wall'
 
     #include <stdlib.h>
 
     #include <stdlib.h>
 
     #include <stdio.h>
 
     #include <stdio.h>
     #include <sys/mman.h> // Needed for mlockall()
+
     #include <sys/mman.h> // Needed for mlockall()
     #include <unistd.h> // needed for sysconf(int name);
+
     #include <unistd.h> // needed for sysconf(int name);
 
     #include <malloc.h>
 
     #include <malloc.h>
     #include <sys/time.h> // needed for getrusage
+
     #include <sys/time.h> // needed for getrusage
     #include <sys/resource.h> // needed for getrusage
+
     #include <sys/resource.h> // needed for getrusage
 
     #include <pthread.h>
 
     #include <pthread.h>
 
     #include <limits.h>
 
     #include <limits.h>
 
      
 
      
     // Struct containing all the info about the thread to start
+
     #define PRE_ALLOCATION_SIZE (100*1024*1024) /* 100MB pagefault free buffer */
     struct thread_info
+
     #define MY_STACK_SIZE      (100*1024)      /* 100 kB is enough for now. */
 +
   
 +
    static void setprio(int prio, int sched)
 
     {
 
     {
        void* (*thread)(void* args); // Routine name
+
    struct sched_param param;
        void* args;                  // Arguments to pass to the thread
+
    // Set realtime priority for this thread
     };
+
    param.sched_priority = prio;
 +
    if (sched_setscheduler(0, sched, &param) < 0)
 +
    perror("sched_setscheduler");
 +
     }
 
      
 
      
     #define PRE_ALLOCATION_SIZE (100*1024*1024) // 100MB pagefault free buffer
+
     void show_new_pagefault_count(const char* logtext,
    #define MY_STACK_SIZE      (100*1024)     // 100 kB is enough for now.
+
          const char* allowed_maj,
 +
          const char* allowed_min)
 +
    {
 +
    static int last_majflt = 0, last_minflt = 0;
 +
    struct rusage usage;
 
      
 
      
    /*************************************************************/
+
    getrusage(RUSAGE_SELF, &usage);
    /* The thread to start */
+
    static int mydata = 12345;
+
 
      
 
      
     static void* my_rt_thread(void* args)
+
    printf("%-30.30s: Pagefaults, Major:%ld (Allowed %s), " \
 +
          "Minor:%ld (Allowed %s)\n", logtext,
 +
          usage.ru_majflt - last_majflt, allowed_maj,
 +
          usage.ru_minflt - last_minflt, allowed_min);
 +
   
 +
    last_majflt = usage.ru_majflt;
 +
    last_minflt = usage.ru_minflt;
 +
    }
 +
   
 +
     static void prove_thread_stack_use_is_safe(int stacksize)
 
     {
 
     {
        struct timespec ts;
+
    volatile char buffer[stacksize];
        ts.tv_sec = 30;
+
    int i;
        ts.tv_nsec = 0;
+
   
       
+
    /* Prove that this thread is behaving well */
        printf("I am an RT-thread with a stack that does not generate page-faults during use, data=%i\n", *((int*)args));
+
    for (i = 0; i < stacksize; i += sysconf(_SC_PAGESIZE)) {
       
+
    /* Each write to this buffer shall NOT generate a
        //<do your RT-thing>
+
    pagefault. */
       
+
    buffer[i] = i;
        clock_nanosleep(CLOCK_REALTIME, 0, &ts, NULL); // wait 30 seconds before thread terminates
+
    }
       
+
   
        return NULL;
+
    show_new_pagefault_count("Caused by using thread stack", "0", "0");
 
     }
 
     }
    /*************************************************************/
 
 
      
 
      
     static void* rt_thread_wrapper(void* args)
+
    /*************************************************************/
 +
    /* The thread to start */
 +
     static void *my_rt_thread(void *args)
 
     {
 
     {
        void* result;
+
    struct timespec ts;
        struct thread_info* thread_info = (struct thread_info*)args;
+
    ts.tv_sec = 30;
       
+
    ts.tv_nsec = 0;
        { // limit the scope to make sure the next temporary variables are released after touching the stack.
+
            int cntr;
+
            // Calculate the size of the stack that is free after this line
+
            volatile int my_size = MY_STACK_SIZE - (sizeof(void*) + \
+
                                                    sizeof(struct thread_info*) + \
+
                                                    sizeof(int) + sizeof(volatile int));
+
            volatile char pretouch_buffer[my_size]; // Use an automatic array that claims the remaining part of the stack
+
 
      
 
      
            // Touch each page on the stack to make sure it is in RAM
+
    setprio(sched_get_priority_max(SCHED_RR), SCHED_RR);
            for (cntr = 0; cntr < my_size; cntr++)
+
            {
+
            pretouch_buffer[cntr] = cntr;
+
            }
+
        }
+
 
      
 
      
        { // limit the scope.
+
    printf("I am an RT-thread with a stack that does not generate " \
            struct sched_param param;
+
          "page-faults during use, stacksize=%i\n", MY_STACK_SIZE);
            // Set realtime priority for this thread
+
            param.sched_priority = sched_get_priority_max(SCHED_RR);
+
            if (sched_setscheduler(0, SCHED_RR, &param) < 0)
+
            {
+
                perror("sched_setscheduler");
+
            }
+
        }
+
 
      
 
      
        // Execute the thread with the proper arguments.
+
    //<do your RT-thing here>
        result = (*thread_info->thread)(thread_info->args);
+
   
       
+
    show_new_pagefault_count("Caused by creating thread", ">=0", ">=0");
        // Release the memory allocated for args
+
   
        if (args) free(args);
+
    prove_thread_stack_use_is_safe(MY_STACK_SIZE);
        return result;
+
   
 +
    /* wait 30 seconds before thread terminates */
 +
    clock_nanosleep(CLOCK_REALTIME, 0, &ts, NULL);
 +
   
 +
    return NULL;
 
     }
 
     }
 +
   
 +
    /*************************************************************/
 
      
 
      
 
     static void error(int at)
 
     static void error(int at)
 
     {
 
     {
        // Just exit on error
+
    /* Just exit on error */
        fprintf(stderr, "Some error occured at %d", at);
+
    fprintf(stderr, "Some error occured at %d", at);
        exit(1);
+
    exit(1);
 
     }
 
     }
 
      
 
      
 
     static void start_rt_thread(void)
 
     static void start_rt_thread(void)
 
     {
 
     {
        // thread_info is freed when the thread terminates.
+
    pthread_t thread;
        struct thread_info* thread_info = malloc(sizeof(struct thread_info));
+
    pthread_attr_t attr;
 
      
 
      
        if (thread_info)
+
    /* init to default values */
        {
+
    if (pthread_attr_init(&attr))
            pthread_t          thread;
+
    error(1);
            pthread_attr_t      attr;
+
    /* Set the requested stacksize for this thread */
 +
    if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN + MY_STACK_SIZE))
 +
    error(2);
 +
    /* And finally start the actual thread */
 +
    pthread_create(&thread, &attr, my_rt_thread, NULL);
 +
    }
 +
   
 +
    static void configure_malloc_behavior(void)
 +
    {
 +
    /* Now lock all current and future pages
 +
      from preventing of being paged */
 +
    if (mlockall(MCL_CURRENT | MCL_FUTURE))
 +
    perror("mlockall failed:");
 
      
 
      
            thread_info->thread = my_rt_thread;
+
    /* Turn off malloc trimming.*/
            thread_info->args  = &mydata;
+
    mallopt(M_TRIM_THRESHOLD, -1);
 
      
 
      
            // init to default values
+
    /* Turn off mmap usage. */
            if (pthread_attr_init(&attr)) error(1);
+
    mallopt(M_MMAP_MAX, 0);
            // Set the requested stacksize for this thread
+
            if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN + MY_STACK_SIZE)) error(2);
+
            // And finally start the actual thread
+
            pthread_create(&thread, &attr, rt_thread_wrapper, thread_info);
+
        }
+
        else
+
        {
+
            error(3);
+
        }
+
 
     }
 
     }
 
      
 
      
     int main(int argc, char* argv[])
+
     static void reserve_process_memory(int size)
 
     {
 
     {
        // Allocate some memory
+
    int i;
        int i, page_size;
+
    char *buffer;
        char* buffer;
+
   
        struct rusage usage;
+
    buffer = malloc(size);
       
+
   
        // Now lock all current and future pages from preventing of being paged
+
    /* Touch each page in this piece of memory to get it mapped into RAM */
        if (mlockall(MCL_CURRENT | MCL_FUTURE ))
+
    for (i = 0; i < size; i += sysconf(_SC_PAGESIZE)) {
        {
+
    /* Each write to this buffer will generate a pagefault.
            perror("mlockall failed:");
+
      Once the pagefault is handled a page will be locked in
        }
+
      memory and never given back to the system. */
       
+
    buffer[i] = 0;
        // Turn off malloc trimming.
+
    }
        mallopt (M_TRIM_THRESHOLD, -1);
+
   
       
+
    /* buffer will now be released. As Glibc is configured such that it  
        // Turn off mmap usage.
+
      never gives back memory to the kernel, the memory allocated above is
        mallopt (M_MMAP_MAX, 0);
+
      locked for this process. All malloc() and new() calls come from
       
+
      the memory pool reserved and locked above. Issuing free() and
        page_size = sysconf(_SC_PAGESIZE);
+
      delete() does NOT make this locking undone. So, with this locking
        buffer = malloc(PRE_ALLOCATION_SIZE);
+
      mechanism we can build C++ applications that will never run into
       
+
      a major/minor pagefault, even with swapping enabled. */
        // Touch each page in this piece of memory to get it mapped into RAM
+
    free(buffer);
        for (i=0; i < PRE_ALLOCATION_SIZE; i+=page_size)
+
        {
+
            // Each write to this buffer will generate a pagefault.
+
            // Once the pagefault is handled a page will be locked in memory and never
+
            // given back to the system.
+
            buffer[i] = 0;
+
            // print the number of major and minor pagefaults this application has triggered
+
            getrusage(RUSAGE_SELF, &usage);
+
            printf("Major-pagefaults:%ld, Minor Pagefaults:%ld\n", usage.ru_majflt, usage.ru_minflt);
+
        }
+
        free(buffer);
+
        printf("Look at the output of ps -leyf, and see that the RSS is now about %d [MB]\n", PRE_ALLOCATION_SIZE/(1024*1024));
+
        // buffer is now released. As Glibc is configured such that it never gives back memory to
+
        // the kernel, the memory allocated above is locked for this process. All malloc() and new()
+
        // calls come from the memory pool reserved and locked above. Issuing free() and delete()
+
        // does NOT make this locking undone. So, with this locking mechanism we can build C++ applications
+
        // that will never run into a major/minor pagefault, even with swapping enabled.
+
       
+
        start_rt_thread();
+
       
+
        //<do your RT-thing>
+
       
+
        printf("Press <ENTER> to exit\n");
+
        getc(stdin);
+
       
+
        return 0;
+
 
     }
 
     }
 
+
   
 +
    int main(int argc, char *argv[])
 +
    {
 +
    show_new_pagefault_count("Initial count", ">=0", ">=0");
 +
   
 +
    configure_malloc_behavior();
 +
   
 +
    show_new_pagefault_count("mlockall() generated", ">=0", ">=0");
 +
   
 +
    reserve_process_memory(PRE_ALLOCATION_SIZE);
 +
   
 +
    show_new_pagefault_count("malloc() and touch generated",
 +
    ">=0", ">=0");
 +
   
 +
    /* Now allocate the memory for the 2nd time and prove the number of
 +
      pagefaults are zero */
 +
    reserve_process_memory(PRE_ALLOCATION_SIZE);
 +
    show_new_pagefault_count("2nd malloc() and use generated",
 +
    "0", "0");
 +
   
 +
    printf("\n\nLook at the output of ps -leyf, and see that the " \
 +
          "RSS is now about %d [MB]\n",
 +
          PRE_ALLOCATION_SIZE / (1024 * 1024));
 +
   
 +
    start_rt_thread();
 +
   
 +
    //<do your RT-thing>
 +
   
 +
    printf("Press <ENTER> to exit\n");
 +
    getc(stdin);
 +
   
 +
    return 0;
 +
    }
 +
   
 
<BR>
 
<BR>
 
The following program verifies the previous statements regarding the effects of the mlockall() function on stack memory.
 
The following program verifies the previous statements regarding the effects of the mlockall() function on stack memory.
Line 179: Line 191:
 
! align="left" valign="top" colspan="2" | <b>Revision History</b>
 
! align="left" valign="top" colspan="2" | <b>Revision History</b>
 
|-  
 
|-  
| align="left" | Revision 1
+
| align="left" | Revision 2
 
| align="left" | 2008-01-15
 
| align="left" | 2008-01-15
 
|}
 
|}

Revision as of 21:57, 15 January 2008

Threaded RT-application with memory locking and stack touching example

   // Compile with 'gcc thisfile.c -lrt -Wall'
   #include <stdlib.h>
   #include <stdio.h>
   #include <sys/mman.h>	// Needed for mlockall()
   #include <unistd.h>		// needed for sysconf(int name);
   #include <malloc.h>
   #include <sys/time.h>	// needed for getrusage
   #include <sys/resource.h>	// needed for getrusage
   #include <pthread.h>
   #include <limits.h>
   
   #define PRE_ALLOCATION_SIZE (100*1024*1024) /* 100MB pagefault free buffer */
   #define MY_STACK_SIZE       (100*1024)      /* 100 kB is enough for now. */
   
   static void setprio(int prio, int sched)
   {
   	struct sched_param param;
   	// Set realtime priority for this thread
   	param.sched_priority = prio;
   	if (sched_setscheduler(0, sched, &param) < 0)
   		perror("sched_setscheduler");
   }
   
   void show_new_pagefault_count(const char* logtext, 
   			      const char* allowed_maj,
   			      const char* allowed_min)
   {
   	static int last_majflt = 0, last_minflt = 0;
   	struct rusage usage;
   
   	getrusage(RUSAGE_SELF, &usage);
   
   	printf("%-30.30s: Pagefaults, Major:%ld (Allowed %s), " \
   	       "Minor:%ld (Allowed %s)\n", logtext,
   	       usage.ru_majflt - last_majflt, allowed_maj,
   	       usage.ru_minflt - last_minflt, allowed_min);
   	
   	last_majflt = usage.ru_majflt; 
   	last_minflt = usage.ru_minflt;
   }
   
   static void prove_thread_stack_use_is_safe(int stacksize)
   {
   	volatile char buffer[stacksize];
   	int i;
   
   	/* Prove that this thread is behaving well */
   	for (i = 0; i < stacksize; i += sysconf(_SC_PAGESIZE)) {
   		/* Each write to this buffer shall NOT generate a 
   			pagefault. */
   		buffer[i] = i;
   	}
   
   	show_new_pagefault_count("Caused by using thread stack", "0", "0");
   }
   
   /*************************************************************/
   /* The thread to start */
   static void *my_rt_thread(void *args)
   {
   	struct timespec ts;
   	ts.tv_sec = 30;
   	ts.tv_nsec = 0;
   
   	setprio(sched_get_priority_max(SCHED_RR), SCHED_RR);
   
   	printf("I am an RT-thread with a stack that does not generate " \
   	       "page-faults during use, stacksize=%i\n", MY_STACK_SIZE);
   
   //<do your RT-thing here>
   
   	show_new_pagefault_count("Caused by creating thread", ">=0", ">=0");
   
   	prove_thread_stack_use_is_safe(MY_STACK_SIZE);
   
   	/* wait 30 seconds before thread terminates */
   	clock_nanosleep(CLOCK_REALTIME, 0, &ts, NULL);
   
   	return NULL;
   }
   
   /*************************************************************/
   
   static void error(int at)
   {
   	/* Just exit on error */
   	fprintf(stderr, "Some error occured at %d", at);
   	exit(1);
   }
   
   static void start_rt_thread(void)
   {
   	pthread_t thread;
   	pthread_attr_t attr;
   
   	/* init to default values */
   	if (pthread_attr_init(&attr))
   		error(1);
   	/* Set the requested stacksize for this thread */
   	if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN + MY_STACK_SIZE))
   		error(2);
   	/* And finally start the actual thread */
   	pthread_create(&thread, &attr, my_rt_thread, NULL);
   }
   
   static void configure_malloc_behavior(void)
   {
   	/* Now lock all current and future pages 
   	   from preventing of being paged */
   	if (mlockall(MCL_CURRENT | MCL_FUTURE))
   		perror("mlockall failed:");
   
   	/* Turn off malloc trimming.*/
   	mallopt(M_TRIM_THRESHOLD, -1);
   
   	/* Turn off mmap usage. */
   	mallopt(M_MMAP_MAX, 0);
   }
   
   static void reserve_process_memory(int size)
   {
   	int i;
   	char *buffer;
   
   	buffer = malloc(size);
   
   	/* Touch each page in this piece of memory to get it mapped into RAM */
   	for (i = 0; i < size; i += sysconf(_SC_PAGESIZE)) {
   		/* Each write to this buffer will generate a pagefault.
   		   Once the pagefault is handled a page will be locked in
   		   memory and never given back to the system. */
   		buffer[i] = 0;
   	}
   
   	/* buffer will now be released. As Glibc is configured such that it 
   	   never gives back memory to the kernel, the memory allocated above is
   	   locked for this process. All malloc() and new() calls come from
   	   the memory pool reserved and locked above. Issuing free() and
   	   delete() does NOT make this locking undone. So, with this locking
   	   mechanism we can build C++ applications that will never run into
   	   a major/minor pagefault, even with swapping enabled. */
   	free(buffer);
   }
   
   int main(int argc, char *argv[])
   {
   	show_new_pagefault_count("Initial count", ">=0", ">=0");
   
   	configure_malloc_behavior();
   
   	show_new_pagefault_count("mlockall() generated", ">=0", ">=0");
   
   	reserve_process_memory(PRE_ALLOCATION_SIZE);
   
   	show_new_pagefault_count("malloc() and touch generated", 
   				 ">=0", ">=0");
   
   	/* Now allocate the memory for the 2nd time and prove the number of
   	   pagefaults are zero */
   	reserve_process_memory(PRE_ALLOCATION_SIZE);
   	show_new_pagefault_count("2nd malloc() and use generated", 
   				 "0", "0");
   
   	printf("\n\nLook at the output of ps -leyf, and see that the " \
   	       "RSS is now about %d [MB]\n",
   	       PRE_ALLOCATION_SIZE / (1024 * 1024));
   
   	start_rt_thread();
   
   //<do your RT-thing>
   
   	printf("Press <ENTER> to exit\n");
   	getc(stdin);
   
   	return 0;
   }
   


The following program verifies the previous statements regarding the effects of the mlockall() function on stack memory. Verifying mlockall() effects on stack memory proof

Author/Maintainer

Remy Bohmer

Revision

Revision History
Revision 2 2008-01-15
Personal tools