/* * * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms * of such GNU licence. * * A copy of the licence is included with the program, and can also be obtained * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301, USA. * */ /* * Register-based HW access backend specific job scheduler APIs */ #include #include #include #include /* * Define for when dumping is enabled. * This should not be based on the instrumentation level as whether dumping is * enabled for a particular level is down to the integrator. However this is * being used for now as otherwise the cinstr headers would be needed. */ #define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL) /* * Hold the runpool_mutex for this */ static inline bool timer_callback_should_run(struct kbase_device *kbdev) { struct kbase_backend_data *backend = &kbdev->hwaccess.backend; s8 nr_running_ctxs; lockdep_assert_held(&kbdev->js_data.runpool_mutex); /* Timer must stop if we are suspending */ if (backend->suspend_timer) return false; /* nr_contexts_pullable is updated with the runpool_mutex. However, the * locking in the caller gives us a barrier that ensures * nr_contexts_pullable is up-to-date for reading */ nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); #ifdef CONFIG_MALI_DEBUG if (kbdev->js_data.softstop_always) { /* Debug support for allowing soft-stop on a single context */ return true; } #endif /* CONFIG_MALI_DEBUG */ if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) { /* Timeouts would have to be 4x longer (due to micro- * architectural design) to support OpenCL conformance tests, so * only run the timer when there's: * - 2 or more CL contexts * - 1 or more GLES contexts * * NOTE: We will treat a context that has both Compute and Non- * Compute jobs will be treated as an OpenCL context (hence, we * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE). */ { s8 nr_compute_ctxs = kbasep_js_ctx_attr_count_on_runpool(kbdev, KBASEP_JS_CTX_ATTR_COMPUTE); s8 nr_noncompute_ctxs = nr_running_ctxs - nr_compute_ctxs; return (bool) (nr_compute_ctxs >= 2 || nr_noncompute_ctxs > 0); } } else { /* Run the timer callback whenever you have at least 1 context */ return (bool) (nr_running_ctxs > 0); } } static enum hrtimer_restart timer_callback(struct hrtimer *timer) { unsigned long flags; struct kbase_device *kbdev; struct kbasep_js_device_data *js_devdata; struct kbase_backend_data *backend; int s; bool reset_needed = false; KBASE_DEBUG_ASSERT(timer != NULL); backend = container_of(timer, struct kbase_backend_data, scheduling_timer); kbdev = container_of(backend, struct kbase_device, hwaccess.backend); js_devdata = &kbdev->js_data; /* Loop through the slots */ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags); for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) { struct kbase_jd_atom *atom = NULL; if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) { atom = kbase_gpu_inspect(kbdev, s, 0); KBASE_DEBUG_ASSERT(atom != NULL); } if (atom != NULL) { /* The current version of the model doesn't support * Soft-Stop */ if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { u32 ticks = atom->sched_info.cfs.ticks++; #if !CINSTR_DUMPING_ENABLED u32 soft_stop_ticks, hard_stop_ticks, gpu_reset_ticks; if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) { soft_stop_ticks = js_devdata->soft_stop_ticks_cl; hard_stop_ticks = js_devdata->hard_stop_ticks_cl; gpu_reset_ticks = js_devdata->gpu_reset_ticks_cl; } else { soft_stop_ticks = js_devdata->soft_stop_ticks; hard_stop_ticks = js_devdata->hard_stop_ticks_ss; gpu_reset_ticks = js_devdata->gpu_reset_ticks_ss; } /* Job is Soft-Stoppable */ if (ticks == soft_stop_ticks) { int disjoint_threshold = KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD; u32 softstop_flags = 0u; /* Job has been scheduled for at least * js_devdata->soft_stop_ticks ticks. * Soft stop the slot so we can run * other jobs. */ dev_dbg(kbdev->dev, "Soft-stop"); #if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS /* nr_user_contexts_running is updated * with the runpool_mutex, but we can't * take that here. * * However, if it's about to be * increased then the new context can't * run any jobs until they take the * runpool_irq lock, so it's OK to * observe the older value. * * Similarly, if it's about to be * decreased, the last job from another * context has already finished, so it's * not too bad that we observe the older * value and register a disjoint event * when we try soft-stopping */ if (js_devdata->nr_user_contexts_running >= disjoint_threshold) softstop_flags |= JS_COMMAND_SW_CAUSES_DISJOINT; kbase_job_slot_softstop_swflags(kbdev, s, atom, softstop_flags); #endif } else if (ticks == hard_stop_ticks) { /* Job has been scheduled for at least * js_devdata->hard_stop_ticks_ss ticks. * It should have been soft-stopped by * now. Hard stop the slot. */ #if !KBASE_DISABLE_SCHEDULING_HARD_STOPS int ms = js_devdata->scheduling_period_ns / 1000000u; dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", (unsigned long)ticks, (unsigned long)ms); kbase_job_slot_hardstop(atom->kctx, s, atom); #endif } else if (ticks == gpu_reset_ticks) { /* Job has been scheduled for at least * js_devdata->gpu_reset_ticks_ss ticks. * It should have left the GPU by now. * Signal that the GPU needs to be * reset. */ reset_needed = true; } #else /* !CINSTR_DUMPING_ENABLED */ /* NOTE: During CINSTR_DUMPING_ENABLED, we use * the alternate timeouts, which makes the hard- * stop and GPU reset timeout much longer. We * also ensure that we don't soft-stop at all. */ if (ticks == js_devdata->soft_stop_ticks) { /* Job has been scheduled for at least * js_devdata->soft_stop_ticks. We do * not soft-stop during * CINSTR_DUMPING_ENABLED, however. */ dev_dbg(kbdev->dev, "Soft-stop"); } else if (ticks == js_devdata->hard_stop_ticks_dumping) { /* Job has been scheduled for at least * js_devdata->hard_stop_ticks_dumping * ticks. Hard stop the slot. */ #if !KBASE_DISABLE_SCHEDULING_HARD_STOPS int ms = js_devdata->scheduling_period_ns / 1000000u; dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", (unsigned long)ticks, (unsigned long)ms); kbase_job_slot_hardstop(atom->kctx, s, atom); #endif } else if (ticks == js_devdata->gpu_reset_ticks_dumping) { /* Job has been scheduled for at least * js_devdata->gpu_reset_ticks_dumping * ticks. It should have left the GPU by * now. Signal that the GPU needs to be * reset. */ reset_needed = true; } #endif /* !CINSTR_DUMPING_ENABLED */ } } } #if KBASE_GPU_RESET_EN if (reset_needed) { dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve."); if (kbase_prepare_to_reset_gpu_locked(kbdev)) kbase_reset_gpu_locked(kbdev); } #endif /* KBASE_GPU_RESET_EN */ /* the timer is re-issued if there is contexts in the run-pool */ if (backend->timer_running) hrtimer_start(&backend->scheduling_timer, HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), HRTIMER_MODE_REL); spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags); return HRTIMER_NORESTART; } void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) { struct kbasep_js_device_data *js_devdata = &kbdev->js_data; struct kbase_backend_data *backend = &kbdev->hwaccess.backend; unsigned long flags; lockdep_assert_held(&js_devdata->runpool_mutex); if (!timer_callback_should_run(kbdev)) { /* Take spinlock to force synchronisation with timer */ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags); backend->timer_running = false; spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags); /* From now on, return value of timer_callback_should_run() will * also cause the timer to not requeue itself. Its return value * cannot change, because it depends on variables updated with * the runpool_mutex held, which the caller of this must also * hold */ hrtimer_cancel(&backend->scheduling_timer); } if (timer_callback_should_run(kbdev) && !backend->timer_running) { /* Take spinlock to force synchronisation with timer */ spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags); backend->timer_running = true; spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags); hrtimer_start(&backend->scheduling_timer, HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns), HRTIMER_MODE_REL); KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u); } } int kbase_backend_timer_init(struct kbase_device *kbdev) { struct kbase_backend_data *backend = &kbdev->hwaccess.backend; hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); backend->scheduling_timer.function = timer_callback; backend->timer_running = false; return 0; } void kbase_backend_timer_term(struct kbase_device *kbdev) { struct kbase_backend_data *backend = &kbdev->hwaccess.backend; hrtimer_cancel(&backend->scheduling_timer); } void kbase_backend_timer_suspend(struct kbase_device *kbdev) { struct kbase_backend_data *backend = &kbdev->hwaccess.backend; backend->suspend_timer = true; kbase_backend_ctx_count_changed(kbdev); } void kbase_backend_timer_resume(struct kbase_device *kbdev) { struct kbase_backend_data *backend = &kbdev->hwaccess.backend; backend->suspend_timer = false; kbase_backend_ctx_count_changed(kbdev); }