Merge branch 'qu6-rel-topic' into uek-build

Author: jedlickat

arch/sparc/kernel/fbt_blacklist.h

@@ -28,3 +28,9 @@ BL_DENTRY(void *, gup_pud_range)
 BL_DENTRY(void *, gup_pmd_range)
 BL_DENTRY(void *, gup_huge_pmd)
 BL_DENTRY(void *, gup_pte_range)
+
+/*
+ * Functions used in dtrace_sync().
+ */
+BL_DENTRY(void *, find_next_bit)
+BL_DENTRY(void *, _find_next_bit)

arch/x86/kernel/fbt_blacklist.h

@@ -66,3 +66,9 @@ BL_DENTRY(void *, gup_pmd_range)
 BL_DENTRY(void *, gup_huge_pmd)
 BL_DENTRY(void *, gup_pte_range)
 BL_DENTRY(void *, pte_mfn_to_pfn)
+
+/*
+ * Functions used in dtrace_sync().
+ */
+BL_DENTRY(void *, find_next_bit)
+BL_DENTRY(void *, _find_next_bit)

dtrace/dtrace_ecb.c

@@ -77,6 +77,41 @@ static dtrace_action_t *dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb,
 		break;
 	}
 
+	case DTRACEAGG_LLQUANTIZE: {
+		uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(desc->dtad_arg);
+		uint16_t lmag = DTRACE_LLQUANTIZE_LMAG(desc->dtad_arg);
+		uint16_t hmag = DTRACE_LLQUANTIZE_HMAG(desc->dtad_arg);
+		uint16_t steps = DTRACE_LLQUANTIZE_STEPS(desc->dtad_arg);
+
+		agg->dtag_initial = desc->dtad_arg;
+		agg->dtag_aggregate = dtrace_aggregate_llquantize;
+
+		/*
+		 * 64 is the largest hmag can practically be (for the smallest
+		 * possible value of factor, 2).  libdtrace has already checked
+		 * for overflow, so if hmag > 64, we have corrupted DOF.
+		 */
+		if (factor < 2 || steps == 0 || hmag > 64)
+			goto err;
+
+		/*
+		 * The size of the buffer for an llquantize() is given by:
+		 *   (hmag-lmag+1) logarithmic ranges
+		 *   x
+		 *   (steps - steps/factor) bins per range
+		 *   x
+		 *   2 signs
+		 *   +
+		 *   two overflow bins
+		 *   +
+		 *   one underflow bin
+		 *   +
+		 *   beginning word to encode factor,lmag,hmag,steps
+		 */
+		size = ((hmag-lmag+1)*(steps-steps/factor)*2+4) * sizeof (uint64_t);
+		break;
+	}
+
 	case DTRACEAGG_AVG:
 		agg->dtag_aggregate = dtrace_aggregate_avg;
 		size = sizeof(uint64_t) * 2;
@@ -129,11 +164,7 @@ static dtrace_action_t *dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb,
 	/*
 	 * Get an ID for the aggregation (add it to the idr).
 	 */
-	mutex_unlock(&dtrace_lock);
-
 	idr_preload(GFP_KERNEL);
-	mutex_lock(&dtrace_lock);
-
 	aggid = idr_alloc_cyclic(&state->dts_agg_idr, agg, 0, 0, GFP_NOWAIT);
 	idr_preload_end();
 	if (aggid < 0) {
@@ -222,6 +253,7 @@ static int dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
 						state, (char *)(uintptr_t)arg);
 			}
 
+		case DTRACEACT_TRACEMEM:
 		case DTRACEACT_LIBACT:
 		case DTRACEACT_DIFEXPR:
 			if (dp == NULL)
@@ -547,6 +579,11 @@ static dtrace_ecb_t *dtrace_ecb_add(dtrace_state_t *state,
 		}
 
 		ecbs = vzalloc(necbs * sizeof(*ecbs));
+		if (ecbs == NULL) {
+			kfree(ecb);
+			return NULL;
+		}
+
 		if (oecbs != NULL)
 			memcpy(ecbs, oecbs, state->dts_necbs * sizeof(*ecbs));
 
@@ -591,6 +628,9 @@ static dtrace_ecb_t *dtrace_ecb_create(dtrace_state_t *state,
 	ASSERT(state != NULL);
 
 	ecb = dtrace_ecb_add(state, probe);
+	if (ecb == NULL)
+		return NULL;
+
 	ecb->dte_uarg = desc->dted_uarg;
 
 	if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) {

dtrace/dtrace_isa.c

@@ -20,6 +20,7 @@
 #include <linux/mm.h>
 #include <linux/smp.h>
 #include <linux/uaccess.h>
+#include <linux/cpumask.h>
 #include <asm/cacheflush.h>
 #include <asm/ptrace.h>
 #include <asm/stacktrace.h>
@@ -36,10 +37,6 @@ int dtrace_getipl(void)
 	return in_interrupt();
 }
 
-static void dtrace_sync_func(void)
-{
-}
-
 void dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg)
 {
 	if (cpu == DTRACE_CPUALL) {
@@ -48,14 +45,133 @@ void dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg)
 		smp_call_function_single(cpu, func, arg, 1);
 }
 
-void dtrace_sync(void)
+void dtrace_toxic_ranges(void (*func)(uintptr_t, uintptr_t))
 {
-	dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL);
+	/* FIXME */
 }
 
-void dtrace_toxic_ranges(void (*func)(uintptr_t, uintptr_t))
+/*
+ * Note:  not called from probe context.  This function is called
+ * asynchronously (and at a regular interval) from outside of probe context
+ * by the DTrace framework to sync shared data which DTrace probe context
+ * may access without locks.
+ *
+ * Whenever the framework updates data which can be accessed from probe context,
+ * the framework then calls dtrace_sync().  dtrace_sync() guarantees all probes
+ * are using the new data before returning.
+ *
+ * See the comment in dtrace_impl.h which describes this algorithm.
+ * The cpuc_in_probe_ctxt flag is an increasing 16-bit count.  It is odd when
+ * in DTrace probe context and even when not in DTrace probe context.
+ * The upper 15 bits are a counter which are incremented when exiting DTrace
+ * probe context.  These upper 15 bits are used to detect "sample aliasing":
+ * i.e. the target CPU is not in DTrace probe context between samples but
+ * continually enters probe context just before being sampled.
+ *
+ * dtrace_sync() loops over NCPUs.  CPUs which are not in DTrace probe context
+ * (cpuc_in_probe_ctxt is even) are removed from the list.  This is repeated
+ * until there are no CPUs left in the sync list.
+ *
+ * In the rare cases where dtrace_sync() loops over all NCPUs more than
+ * dtrace_sync_sample_count times, dtrace_sync() then spins on one CPU's
+ * cpuc_in_probe_ctxt count until the count increments.  This is intended to
+ * avoid sample aliasing.
+ */
+void dtrace_sync(void)
 {
-	/* FIXME */
+	/*
+	 * sync_cpus is a bitmap of CPUs that need to be synced with.
+	 */
+	cpumask_t	sync_cpus;
+	uint64_t	sample_count = 0;
+	int		cpuid, sample_cpuid;
+	int		outstanding;
+
+	/*
+	 * Create bitmap of CPUs that need to be synced with.
+	 */
+	cpumask_copy(&sync_cpus, cpu_online_mask);
+	outstanding = 0;
+	for_each_cpu(cpuid, &sync_cpus) {
+		++outstanding;
+
+		/*
+		 * Set a flag to let the CPU know we are syncing with it.
+		 */
+		DTRACE_SYNC_START(cpuid);
+	}
+
+	/*
+	 * The preceding stores by DTRACE_SYNC_START() must complete before
+	 * subsequent loads or stores.  No membar is needed because the
+	 * atomic-add operation in DTRACE_SYNC_START is a memory barrier on
+	 * SPARC and X86.
+	 */
+
+	while (outstanding > 0) {
+		/*
+		 * Loop over the map of CPUs that need to be synced with.
+		 */
+		for_each_cpu(cpuid, &sync_cpus) {
+			if (!DTRACE_SYNC_IN_CRITICAL(cpuid)) {
+
+				/* Clear the CPU's sync request flag */
+				DTRACE_SYNC_END(cpuid);
+
+				/*
+				 * remove cpuid from list of CPUs that
+				 * still need to be synced with.
+				 */
+				DTRACE_SYNC_DONE(cpuid, &sync_cpus);
+				--outstanding;
+			} else {
+				/*
+				 * Remember one of the outstanding CPUs to spin
+				 * on once we reach the sampling limit.
+				 */
+				sample_cpuid = cpuid;
+			}
+		}
+
+		/*
+		 * dtrace_probe may be running in sibling threads in this core.
+		 */
+		if (outstanding > 0) {
+			dtrace_safe_smt_pause();
+
+			/*
+			 * After sample_count loops, spin on one CPU's count
+			 * instead of just checking for odd/even.
+			 */
+			if (++sample_count > dtrace_sync_sample_count) {
+				uint64_t count =
+				    DTRACE_SYNC_CRITICAL_COUNT(sample_cpuid);
+
+				/*
+				 * Spin until critical section count increments.
+				 */
+				if (DTRACE_SYNC_IN_CRITICAL(sample_cpuid)) {
+					while (count ==
+					    DTRACE_SYNC_CRITICAL_COUNT(
+					    sample_cpuid)) {
+
+						dtrace_safe_smt_pause();
+					}
+				}
+
+				DTRACE_SYNC_END(sample_cpuid);
+				DTRACE_SYNC_DONE(sample_cpuid, &sync_cpus);
+				--outstanding;
+			}
+		}
+	}
+
+/*
+ * All preceding loads by DTRACE_SYNC_IN_CRITICAL() and
+ * DTRACE_SYNC_CRITICAL_COUNT() must complete before subsequent loads
+ * or stores.  No membar is needed because the atomic-add operation in
+ * DTRACE_SYNC_END() is a memory barrier on SPARC and X86.
+ */
 }
 
 /*

dtrace/dtrace_probe.c

@@ -53,18 +53,24 @@ dtrace_id_t dtrace_probe_create(dtrace_provider_id_t prov, const char *mod,
 	probe = kmem_cache_alloc(dtrace_probe_cachep, __GFP_NOFAIL);
 
 	/*
-	 * The ir_preload() function should be called without holding locks.
+	 * The idr_preload() should be called without holding locks as it may
+	 * block.  At the same time it is required to protect DTrace structures.
+	 * We can't drop it before idr_preload() and acquire after it because
+	 * we can't sleep in atomic context (until we reach idr_preload_end()).
+	 *
+	 * It is better to delay DTrace framework than traced host so the lock
+	 * is being held for the duration of idr allocation.
+	 *
 	 * When the provider is the DTrace core itself, dtrace_lock will be
 	 * held when we enter this function.
 	 */
 	if (provider == dtrace_provider) {
 		ASSERT(MUTEX_HELD(&dtrace_lock));
-		mutex_unlock(&dtrace_lock);
+	} else {
+		mutex_lock(&dtrace_lock);
 	}
 
 	idr_preload(GFP_KERNEL);
-
-	mutex_lock(&dtrace_lock);
 	id = idr_alloc_cyclic(&dtrace_probe_idr, probe, 0, 0, GFP_NOWAIT);
 	idr_preload_end();
 	if (id < 0) {
@@ -515,6 +521,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
 	int			onintr;
 	volatile uint16_t	*flags;
 	int			pflag = 0;
+	uint32_t		re_entry;
 
 #ifdef FIXME
 	/*
@@ -526,12 +533,13 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
 		return;
 #endif
 
+	DTRACE_SYNC_ENTER_CRITICAL(cookie, re_entry);
+
 	/*
 	 * If preemption has already been disabled before we get here, we
 	 * accept it as a free gift.  We just need to make sure that we don't
 	 * re-enable preemption on the way out...
 	 */
-	local_irq_save(cookie);
 	if ((pflag = dtrace_is_preemptive()))
 		dtrace_preempt_off();
 
@@ -547,7 +555,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
 		 */
 		if (pflag)
 			dtrace_preempt_on();
-		local_irq_restore(cookie);
+		DTRACE_SYNC_EXIT_CRITICAL(cookie, re_entry);
 		return;
 	}
 
@@ -557,7 +565,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
 		 */
 		if (pflag)
 			dtrace_preempt_on();
-		local_irq_restore(cookie);
+		DTRACE_SYNC_EXIT_CRITICAL(cookie, re_entry);
 		return;
 	}
 
@@ -574,7 +582,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
 			     pflag);
 		if (pflag)
 			dtrace_preempt_on();
-		local_irq_restore(cookie);
+		DTRACE_SYNC_EXIT_CRITICAL(cookie, re_entry);
 		return;
 	}
 
@@ -996,6 +1004,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
 			case DTRACEACT_PRINTA:
 			case DTRACEACT_SYSTEM:
 			case DTRACEACT_FREOPEN:
+			case DTRACEACT_TRACEMEM:
 				break;
 
 			case DTRACEACT_SYM:
@@ -1249,7 +1258,7 @@ void dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
 
 	if (pflag)
 		dtrace_preempt_on();
-	local_irq_restore(cookie);
+	DTRACE_SYNC_EXIT_CRITICAL(cookie, re_entry);
 
 	if (current->dtrace_sig != 0) {
 		int	sig = current->dtrace_sig;
@@ -1279,16 +1288,7 @@ int dtrace_probe_init(void)
 	 * We need to drop our locks when calling idr_preload(), so we try to
 	 * get them back right after.
 	 */
-	mutex_unlock(&dtrace_lock);
-	mutex_unlock(&dtrace_provider_lock);
-	mutex_unlock(&cpu_lock);
-
 	idr_preload(GFP_KERNEL);
-
-	mutex_lock(&cpu_lock);
-	mutex_lock(&dtrace_provider_lock);
-	mutex_lock(&dtrace_lock);
-
 	id = idr_alloc_cyclic(&dtrace_probe_idr, NULL, 0, 0, GFP_NOWAIT);
 	idr_preload_end();