Logo Search packages:      
Sourcecode: linux version File versions

spu_task_sync.c

/*
 * Cell Broadband Engine OProfile Support
 *
 * (C) Copyright IBM Corporation 2006
 *
 * Author: Maynard Johnson <maynardj@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

/* The purpose of this file is to handle SPU event task switching
 * and to record SPU context information into the OProfile
 * event buffer.
 *
 * Additionally, the spu_sync_buffer function is provided as a helper
 * for recoding actual SPU program counter samples to the event buffer.
 */
#include <linux/dcookies.h>
#include <linux/kref.h>
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/module.h>
#include <linux/notifier.h>
#include <linux/numa.h>
#include <linux/oprofile.h>
#include <linux/spinlock.h>
#include "pr_util.h"

#define RELEASE_ALL 9999

static DEFINE_SPINLOCK(buffer_lock);
static DEFINE_SPINLOCK(cache_lock);
static int num_spu_nodes;
int spu_prof_num_nodes;

struct spu_buffer spu_buff[MAX_NUMNODES * SPUS_PER_NODE];
struct delayed_work spu_work;
static unsigned max_spu_buff;

static void spu_buff_add(unsigned long int value, int spu)
{
      /* spu buff is a circular buffer.  Add entries to the
       * head.  Head is the index to store the next value.
       * The buffer is full when there is one available entry
       * in the queue, i.e. head and tail can't be equal.
       * That way we can tell the difference between the
       * buffer being full versus empty.
       *
       *  ASSUPTION: the buffer_lock is held when this function
       *             is called to lock the buffer, head and tail.
       */
      int full = 1;

      if (spu_buff[spu].head >= spu_buff[spu].tail) {
            if ((spu_buff[spu].head - spu_buff[spu].tail)
                <  (max_spu_buff - 1))
                  full = 0;

      } else if (spu_buff[spu].tail > spu_buff[spu].head) {
            if ((spu_buff[spu].tail - spu_buff[spu].head)
                > 1)
                  full = 0;
      }

      if (!full) {
            spu_buff[spu].buff[spu_buff[spu].head] = value;
            spu_buff[spu].head++;

            if (spu_buff[spu].head >= max_spu_buff)
                  spu_buff[spu].head = 0;
      } else {
            /* From the user's perspective make the SPU buffer
             * size management/overflow look like we are using
             * per cpu buffers.  The user uses the same
             * per cpu parameter to adjust the SPU buffer size.
             * Increment the sample_lost_overflow to inform
             * the user the buffer size needs to be increased.
             */
            oprofile_cpu_buffer_inc_smpl_lost();
      }
}

/* This function copies the per SPU buffers to the
 * OProfile kernel buffer.
 */
void sync_spu_buff(void)
{
      int spu;
      unsigned long flags;
      int curr_head;

      for (spu = 0; spu < num_spu_nodes; spu++) {
            /* In case there was an issue and the buffer didn't
             * get created skip it.
             */
            if (spu_buff[spu].buff == NULL)
                  continue;

            /* Hold the lock to make sure the head/tail
             * doesn't change while spu_buff_add() is
             * deciding if the buffer is full or not.
             * Being a little paranoid.
             */
            spin_lock_irqsave(&buffer_lock, flags);
            curr_head = spu_buff[spu].head;
            spin_unlock_irqrestore(&buffer_lock, flags);

            /* Transfer the current contents to the kernel buffer.
             * data can still be added to the head of the buffer.
             */
            oprofile_put_buff(spu_buff[spu].buff,
                          spu_buff[spu].tail,
                          curr_head, max_spu_buff);

            spin_lock_irqsave(&buffer_lock, flags);
            spu_buff[spu].tail = curr_head;
            spin_unlock_irqrestore(&buffer_lock, flags);
      }

}

static void wq_sync_spu_buff(struct work_struct *work)
{
      /* move data from spu buffers to kernel buffer */
      sync_spu_buff();

      /* only reschedule if profiling is not done */
      if (spu_prof_running)
            schedule_delayed_work(&spu_work, DEFAULT_TIMER_EXPIRE);
}

/* Container for caching information about an active SPU task. */
struct cached_info {
      struct vma_to_fileoffset_map *map;
      struct spu *the_spu;    /* needed to access pointer to local_store */
      struct kref cache_ref;
};

static struct cached_info *spu_info[MAX_NUMNODES * 8];

static void destroy_cached_info(struct kref *kref)
{
      struct cached_info *info;

      info = container_of(kref, struct cached_info, cache_ref);
      vma_map_free(info->map);
      kfree(info);
      module_put(THIS_MODULE);
}

/* Return the cached_info for the passed SPU number.
 * ATTENTION:  Callers are responsible for obtaining the
 *           cache_lock if needed prior to invoking this function.
 */
static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num)
{
      struct kref *ref;
      struct cached_info *ret_info;

      if (spu_num >= num_spu_nodes) {
            printk(KERN_ERR "SPU_PROF: "
                   "%s, line %d: Invalid index %d into spu info cache\n",
                   __func__, __LINE__, spu_num);
            ret_info = NULL;
            goto out;
      }
      if (!spu_info[spu_num] && the_spu) {
            ref = spu_get_profile_private_kref(the_spu->ctx);
            if (ref) {
                  spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref);
                  kref_get(&spu_info[spu_num]->cache_ref);
            }
      }

      ret_info = spu_info[spu_num];
 out:
      return ret_info;
}


/* Looks for cached info for the passed spu.  If not found, the
 * cached info is created for the passed spu.
 * Returns 0 for success; otherwise, -1 for error.
 */
static int
prepare_cached_spu_info(struct spu *spu, unsigned long objectId)
{
      unsigned long flags;
      struct vma_to_fileoffset_map *new_map;
      int retval = 0;
      struct cached_info *info;

      /* We won't bother getting cache_lock here since
       * don't do anything with the cached_info that's returned.
       */
      info = get_cached_info(spu, spu->number);

      if (info) {
            pr_debug("Found cached SPU info.\n");
            goto out;
      }

      /* Create cached_info and set spu_info[spu->number] to point to it.
       * spu->number is a system-wide value, not a per-node value.
       */
      info = kzalloc(sizeof(struct cached_info), GFP_KERNEL);
      if (!info) {
            printk(KERN_ERR "SPU_PROF: "
                   "%s, line %d: create vma_map failed\n",
                   __func__, __LINE__);
            retval = -ENOMEM;
            goto err_alloc;
      }
      new_map = create_vma_map(spu, objectId);
      if (!new_map) {
            printk(KERN_ERR "SPU_PROF: "
                   "%s, line %d: create vma_map failed\n",
                   __func__, __LINE__);
            retval = -ENOMEM;
            goto err_alloc;
      }

      pr_debug("Created vma_map\n");
      info->map = new_map;
      info->the_spu = spu;
      kref_init(&info->cache_ref);
      spin_lock_irqsave(&cache_lock, flags);
      spu_info[spu->number] = info;
      /* Increment count before passing off ref to SPUFS. */
      kref_get(&info->cache_ref);

      /* We increment the module refcount here since SPUFS is
       * responsible for the final destruction of the cached_info,
       * and it must be able to access the destroy_cached_info()
       * function defined in the OProfile module.  We decrement
       * the module refcount in destroy_cached_info.
       */
      try_module_get(THIS_MODULE);
      spu_set_profile_private_kref(spu->ctx, &info->cache_ref,
                        destroy_cached_info);
      spin_unlock_irqrestore(&cache_lock, flags);
      goto out;

err_alloc:
      kfree(info);
out:
      return retval;
}

/*
 * NOTE:  The caller is responsible for locking the
 *      cache_lock prior to calling this function.
 */
static int release_cached_info(int spu_index)
{
      int index, end;

      if (spu_index == RELEASE_ALL) {
            end = num_spu_nodes;
            index = 0;
      } else {
            if (spu_index >= num_spu_nodes) {
                  printk(KERN_ERR "SPU_PROF: "
                        "%s, line %d: "
                        "Invalid index %d into spu info cache\n",
                        __func__, __LINE__, spu_index);
                  goto out;
            }
            end = spu_index + 1;
            index = spu_index;
      }
      for (; index < end; index++) {
            if (spu_info[index]) {
                  kref_put(&spu_info[index]->cache_ref,
                         destroy_cached_info);
                  spu_info[index] = NULL;
            }
      }

out:
      return 0;
}

/* The source code for fast_get_dcookie was "borrowed"
 * from drivers/oprofile/buffer_sync.c.
 */

/* Optimisation. We can manage without taking the dcookie sem
 * because we cannot reach this code without at least one
 * dcookie user still being registered (namely, the reader
 * of the event buffer).
 */
static inline unsigned long fast_get_dcookie(struct path *path)
{
      unsigned long cookie;

      if (path->dentry->d_cookie)
            return (unsigned long)path->dentry;
      get_dcookie(path, &cookie);
      return cookie;
}

/* Look up the dcookie for the task's first VM_EXECUTABLE mapping,
 * which corresponds loosely to "application name". Also, determine
 * the offset for the SPU ELF object.  If computed offset is
 * non-zero, it implies an embedded SPU object; otherwise, it's a
 * separate SPU binary, in which case we retrieve it's dcookie.
 * For the embedded case, we must determine if SPU ELF is embedded
 * in the executable application or another file (i.e., shared lib).
 * If embedded in a shared lib, we must get the dcookie and return
 * that to the caller.
 */
static unsigned long
get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp,
                      unsigned long *spu_bin_dcookie,
                      unsigned long spu_ref)
{
      unsigned long app_cookie = 0;
      unsigned int my_offset = 0;
      struct file *app = NULL;
      struct vm_area_struct *vma;
      struct mm_struct *mm = spu->mm;

      if (!mm)
            goto out;

      down_read(&mm->mmap_sem);

      for (vma = mm->mmap; vma; vma = vma->vm_next) {
            if (!vma->vm_file)
                  continue;
            if (!(vma->vm_flags & VM_EXECUTABLE))
                  continue;
            app_cookie = fast_get_dcookie(&vma->vm_file->f_path);
            pr_debug("got dcookie for %s\n",
                   vma->vm_file->f_dentry->d_name.name);
            app = vma->vm_file;
            break;
      }

      for (vma = mm->mmap; vma; vma = vma->vm_next) {
            if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref)
                  continue;
            my_offset = spu_ref - vma->vm_start;
            if (!vma->vm_file)
                  goto fail_no_image_cookie;

            pr_debug("Found spu ELF at %X(object-id:%lx) for file %s\n",
                   my_offset, spu_ref,
                   vma->vm_file->f_dentry->d_name.name);
            *offsetp = my_offset;
            break;
      }

      *spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path);
      pr_debug("got dcookie for %s\n", vma->vm_file->f_dentry->d_name.name);

      up_read(&mm->mmap_sem);

out:
      return app_cookie;

fail_no_image_cookie:
      up_read(&mm->mmap_sem);

      printk(KERN_ERR "SPU_PROF: "
            "%s, line %d: Cannot find dcookie for SPU binary\n",
            __func__, __LINE__);
      goto out;
}



/* This function finds or creates cached context information for the
 * passed SPU and records SPU context information into the OProfile
 * event buffer.
 */
static int process_context_switch(struct spu *spu, unsigned long objectId)
{
      unsigned long flags;
      int retval;
      unsigned int offset = 0;
      unsigned long spu_cookie = 0, app_dcookie;

      retval = prepare_cached_spu_info(spu, objectId);
      if (retval)
            goto out;

      /* Get dcookie first because a mutex_lock is taken in that
       * code path, so interrupts must not be disabled.
       */
      app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId);
      if (!app_dcookie || !spu_cookie) {
            retval  = -ENOENT;
            goto out;
      }

      /* Record context info in event buffer */
      spin_lock_irqsave(&buffer_lock, flags);
      spu_buff_add(ESCAPE_CODE, spu->number);
      spu_buff_add(SPU_CTX_SWITCH_CODE, spu->number);
      spu_buff_add(spu->number, spu->number);
      spu_buff_add(spu->pid, spu->number);
      spu_buff_add(spu->tgid, spu->number);
      spu_buff_add(app_dcookie, spu->number);
      spu_buff_add(spu_cookie, spu->number);
      spu_buff_add(offset, spu->number);

      /* Set flag to indicate SPU PC data can now be written out.  If
       * the SPU program counter data is seen before an SPU context
       * record is seen, the postprocessing will fail.
       */
      spu_buff[spu->number].ctx_sw_seen = 1;

      spin_unlock_irqrestore(&buffer_lock, flags);
      smp_wmb();  /* insure spu event buffer updates are written */
                  /* don't want entries intermingled... */
out:
      return retval;
}

/*
 * This function is invoked on either a bind_context or unbind_context.
 * If called for an unbind_context, the val arg is 0; otherwise,
 * it is the object-id value for the spu context.
 * The data arg is of type 'struct spu *'.
 */
static int spu_active_notify(struct notifier_block *self, unsigned long val,
                        void *data)
{
      int retval;
      unsigned long flags;
      struct spu *the_spu = data;

      pr_debug("SPU event notification arrived\n");
      if (!val) {
            spin_lock_irqsave(&cache_lock, flags);
            retval = release_cached_info(the_spu->number);
            spin_unlock_irqrestore(&cache_lock, flags);
      } else {
            retval = process_context_switch(the_spu, val);
      }
      return retval;
}

static struct notifier_block spu_active = {
      .notifier_call = spu_active_notify,
};

static int number_of_online_nodes(void)
{
        u32 cpu; u32 tmp;
        int nodes = 0;
        for_each_online_cpu(cpu) {
                tmp = cbe_cpu_to_node(cpu) + 1;
                if (tmp > nodes)
                        nodes++;
        }
        return nodes;
}

static int oprofile_spu_buff_create(void)
{
      int spu;

      max_spu_buff = oprofile_get_cpu_buffer_size();

      for (spu = 0; spu < num_spu_nodes; spu++) {
            /* create circular buffers to store the data in.
             * use locks to manage accessing the buffers
             */
            spu_buff[spu].head = 0;
            spu_buff[spu].tail = 0;

            /*
             * Create a buffer for each SPU.  Can't reliably
             * create a single buffer for all spus due to not
             * enough contiguous kernel memory.
             */

            spu_buff[spu].buff = kzalloc((max_spu_buff
                                    * sizeof(unsigned long)),
                                   GFP_KERNEL);

            if (!spu_buff[spu].buff) {
                  printk(KERN_ERR "SPU_PROF: "
                         "%s, line %d:  oprofile_spu_buff_create "
                   "failed to allocate spu buffer %d.\n",
                         __func__, __LINE__, spu);

                  /* release the spu buffers that have been allocated */
                  while (spu >= 0) {
                        kfree(spu_buff[spu].buff);
                        spu_buff[spu].buff = 0;
                        spu--;
                  }
                  return -ENOMEM;
            }
      }
      return 0;
}

/* The main purpose of this function is to synchronize
 * OProfile with SPUFS by registering to be notified of
 * SPU task switches.
 *
 * NOTE: When profiling SPUs, we must ensure that only
 * spu_sync_start is invoked and not the generic sync_start
 * in drivers/oprofile/oprof.c.      A return value of
 * SKIP_GENERIC_SYNC or SYNC_START_ERROR will
 * accomplish this.
 */
int spu_sync_start(void)
{
      int spu;
      int ret = SKIP_GENERIC_SYNC;
      int register_ret;
      unsigned long flags = 0;

      spu_prof_num_nodes = number_of_online_nodes();
      num_spu_nodes = spu_prof_num_nodes * 8;
      INIT_DELAYED_WORK(&spu_work, wq_sync_spu_buff);

      /* create buffer for storing the SPU data to put in
       * the kernel buffer.
       */
      ret = oprofile_spu_buff_create();
      if (ret)
            goto out;

      spin_lock_irqsave(&buffer_lock, flags);
      for (spu = 0; spu < num_spu_nodes; spu++) {
            spu_buff_add(ESCAPE_CODE, spu);
            spu_buff_add(SPU_PROFILING_CODE, spu);
            spu_buff_add(num_spu_nodes, spu);
      }
      spin_unlock_irqrestore(&buffer_lock, flags);

      for (spu = 0; spu < num_spu_nodes; spu++) {
            spu_buff[spu].ctx_sw_seen = 0;
            spu_buff[spu].last_guard_val = 0;
      }

      /* Register for SPU events  */
      register_ret = spu_switch_event_register(&spu_active);
      if (register_ret) {
            ret = SYNC_START_ERROR;
            goto out;
      }

      pr_debug("spu_sync_start -- running.\n");
out:
      return ret;
}

/* Record SPU program counter samples to the oprofile event buffer. */
void spu_sync_buffer(int spu_num, unsigned int *samples,
                 int num_samples)
{
      unsigned long long file_offset;
      unsigned long flags;
      int i;
      struct vma_to_fileoffset_map *map;
      struct spu *the_spu;
      unsigned long long spu_num_ll = spu_num;
      unsigned long long spu_num_shifted = spu_num_ll << 32;
      struct cached_info *c_info;

      /* We need to obtain the cache_lock here because it's
       * possible that after getting the cached_info, the SPU job
       * corresponding to this cached_info may end, thus resulting
       * in the destruction of the cached_info.
       */
      spin_lock_irqsave(&cache_lock, flags);
      c_info = get_cached_info(NULL, spu_num);
      if (!c_info) {
            /* This legitimately happens when the SPU task ends before all
             * samples are recorded.
             * No big deal -- so we just drop a few samples.
             */
            pr_debug("SPU_PROF: No cached SPU contex "
                    "for SPU #%d. Dropping samples.\n", spu_num);
            goto out;
      }

      map = c_info->map;
      the_spu = c_info->the_spu;
      spin_lock(&buffer_lock);
      for (i = 0; i < num_samples; i++) {
            unsigned int sample = *(samples+i);
            int grd_val = 0;
            file_offset = 0;
            if (sample == 0)
                  continue;
            file_offset = vma_map_lookup( map, sample, the_spu, &grd_val);

            /* If overlays are used by this SPU application, the guard
             * value is non-zero, indicating which overlay section is in
             * use.      We need to discard samples taken during the time
             * period which an overlay occurs (i.e., guard value changes).
             */
            if (grd_val && grd_val != spu_buff[spu_num].last_guard_val) {
                  spu_buff[spu_num].last_guard_val = grd_val;
                  /* Drop the rest of the samples. */
                  break;
            }

            /* We must ensure that the SPU context switch has been written
             * out before samples for the SPU.  Otherwise, the SPU context
             * information is not available and the postprocessing of the
             * SPU PC will fail with no available anonymous map information.
             */
            if (spu_buff[spu_num].ctx_sw_seen)
                  spu_buff_add((file_offset | spu_num_shifted),
                               spu_num);
      }
      spin_unlock(&buffer_lock);
out:
      spin_unlock_irqrestore(&cache_lock, flags);
}


int spu_sync_stop(void)
{
      unsigned long flags = 0;
      int ret;
      int k;

      ret = spu_switch_event_unregister(&spu_active);

      if (ret)
            printk(KERN_ERR "SPU_PROF: "
                   "%s, line %d: spu_switch_event_unregister "    \
                   "returned %d\n",
                   __func__, __LINE__, ret);

      /* flush any remaining data in the per SPU buffers */
      sync_spu_buff();

      spin_lock_irqsave(&cache_lock, flags);
      ret = release_cached_info(RELEASE_ALL);
      spin_unlock_irqrestore(&cache_lock, flags);

      /* remove scheduled work queue item rather then waiting
       * for every queued entry to execute.  Then flush pending
       * system wide buffer to event buffer.
       */
      cancel_delayed_work(&spu_work);

      for (k = 0; k < num_spu_nodes; k++) {
            spu_buff[k].ctx_sw_seen = 0;

            /*
             * spu_sys_buff will be null if there was a problem
             * allocating the buffer.  Only delete if it exists.
             */
            kfree(spu_buff[k].buff);
            spu_buff[k].buff = 0;
      }
      pr_debug("spu_sync_stop -- done.\n");
      return ret;
}


Generated by  Doxygen 1.6.0   Back to index