#include "jit/JSJitFrameIter-inl.h"
#include "jit/BaselineDebugModeOSR.h"
#include "jit/BaselineIC.h"
#include "jit/JitcodeMap.h"
#include "jit/JitFrames.h"
#include "jit/Safepoints.h"
using namespace js;
using namespace js::jit;
JSJitFrameIter::JSJitFrameIter(const JitActivation* activation)
: current_(activation->jsExitFP()),
type_(FrameType::Exit),
resumePCinCurrentFrame_(nullptr),
frameSize_(0),
cachedSafepointIndex_(nullptr),
activation_(activation) {
if (activation_->bailoutData()) {
current_ = activation_->bailoutData()->fp();
frameSize_ = activation_->bailoutData()->topFrameSize();
type_ = FrameType::Bailout;
} else {
MOZ_ASSERT(!TlsContext.get()->inUnsafeCallWithABI);
}
}
JSJitFrameIter::JSJitFrameIter(const JitActivation* activation,
FrameType frameType, uint8_t* fp)
: current_(fp),
type_(frameType),
resumePCinCurrentFrame_(nullptr),
frameSize_(0),
cachedSafepointIndex_(nullptr),
activation_(activation) {
MOZ_ASSERT(type_ == FrameType::JSJitToWasm || type_ == FrameType::Exit);
MOZ_ASSERT(!activation_->bailoutData());
MOZ_ASSERT(!TlsContext.get()->inUnsafeCallWithABI);
}
bool JSJitFrameIter::checkInvalidation() const {
IonScript* dummy;
return checkInvalidation(&dummy);
}
bool JSJitFrameIter::checkInvalidation(IonScript** ionScriptOut) const {
JSScript* script = this->script();
if (isBailoutJS()) {
*ionScriptOut = activation_->bailoutData()->ionScript();
return !script->hasIonScript() || script->ionScript() != *ionScriptOut;
}
uint8_t* returnAddr = resumePCinCurrentFrame();
bool invalidated = !script->hasIonScript() ||
!script->ionScript()->containsReturnAddress(returnAddr);
if (!invalidated) {
return false;
}
int32_t invalidationDataOffset = ((int32_t*)returnAddr)[-1];
uint8_t* ionScriptDataOffset = returnAddr + invalidationDataOffset;
IonScript* ionScript = (IonScript*)Assembler::GetPointer(ionScriptDataOffset);
MOZ_ASSERT(ionScript->containsReturnAddress(returnAddr));
*ionScriptOut = ionScript;
return true;
}
CalleeToken JSJitFrameIter::calleeToken() const {
return ((JitFrameLayout*)current_)->calleeToken();
}
JSFunction* JSJitFrameIter::callee() const {
MOZ_ASSERT(isScripted());
MOZ_ASSERT(isFunctionFrame());
return CalleeTokenToFunction(calleeToken());
}
JSFunction* JSJitFrameIter::maybeCallee() const {
if (isScripted() && isFunctionFrame()) {
return callee();
}
return nullptr;
}
bool JSJitFrameIter::isBareExit() const {
if (type_ != FrameType::Exit) {
return false;
}
return exitFrame()->isBareExit();
}
bool JSJitFrameIter::isFunctionFrame() const {
return CalleeTokenIsFunction(calleeToken());
}
JSScript* JSJitFrameIter::script() const {
MOZ_ASSERT(isScripted());
if (isBaselineJS()) {
return baselineFrame()->script();
}
JSScript* script = ScriptFromCalleeToken(calleeToken());
MOZ_ASSERT(script);
return script;
}
void JSJitFrameIter::baselineScriptAndPc(JSScript** scriptRes,
jsbytecode** pcRes) const {
MOZ_ASSERT(isBaselineJS());
JSScript* script = this->script();
if (scriptRes) {
*scriptRes = script;
}
MOZ_ASSERT(pcRes);
if (baselineFrame()->runningInInterpreter()) {
MOZ_ASSERT(baselineFrame()->interpreterScript() == script);
*pcRes = baselineFrame()->interpreterPC();
return;
}
if (jsbytecode* overridePc = baselineFrame()->maybeOverridePc()) {
*pcRes = overridePc;
return;
}
uint8_t* retAddr = resumePCinCurrentFrame();
RetAddrEntry& entry =
script->baselineScript()->retAddrEntryFromReturnAddress(retAddr);
*pcRes = entry.pc(script);
}
Value* JSJitFrameIter::actualArgs() const { return jsFrame()->argv() + 1; }
uint8_t* JSJitFrameIter::prevFp() const {
return current_ + current()->prevFrameLocalSize() + current()->headerSize();
}
void JSJitFrameIter::operator++() {
MOZ_ASSERT(!isEntry());
frameSize_ = prevFrameLocalSize();
cachedSafepointIndex_ = nullptr;
if (isEntry(current()->prevType())) {
type_ = current()->prevType();
return;
}
type_ = current()->prevType();
resumePCinCurrentFrame_ = current()->returnAddress();
current_ = prevFp();
}
uintptr_t* JSJitFrameIter::spillBase() const {
MOZ_ASSERT(isIonJS());
return reinterpret_cast<uintptr_t*>(fp() - ionScript()->frameSize());
}
MachineState JSJitFrameIter::machineState() const {
MOZ_ASSERT(isIonScripted());
if (MOZ_UNLIKELY(isBailoutJS())) {
return *activation_->bailoutData()->machineState();
}
SafepointReader reader(ionScript(), safepoint());
uintptr_t* spill = spillBase();
MachineState machine;
for (GeneralRegisterBackwardIterator iter(reader.allGprSpills()); iter.more();
++iter) {
machine.setRegisterLocation(*iter, --spill);
}
uint8_t* spillAlign =
alignDoubleSpillWithOffset(reinterpret_cast<uint8_t*>(spill), 0);
char* floatSpill = reinterpret_cast<char*>(spillAlign);
FloatRegisterSet fregs = reader.allFloatSpills().set();
fregs = fregs.reduceSetForPush();
for (FloatRegisterBackwardIterator iter(fregs); iter.more(); ++iter) {
floatSpill -= (*iter).size();
for (uint32_t a = 0; a < (*iter).numAlignedAliased(); a++) {
FloatRegister ftmp = (*iter).alignedAliased(a);
machine.setRegisterLocation(ftmp, (double*)floatSpill);
}
}
return machine;
}
JitFrameLayout* JSJitFrameIter::jsFrame() const {
MOZ_ASSERT(isScripted());
if (isBailoutJS()) {
return (JitFrameLayout*)activation_->bailoutData()->fp();
}
return (JitFrameLayout*)fp();
}
IonScript* JSJitFrameIter::ionScript() const {
MOZ_ASSERT(isIonScripted());
if (isBailoutJS()) {
return activation_->bailoutData()->ionScript();
}
IonScript* ionScript = nullptr;
if (checkInvalidation(&ionScript)) {
return ionScript;
}
return ionScriptFromCalleeToken();
}
IonScript* JSJitFrameIter::ionScriptFromCalleeToken() const {
MOZ_ASSERT(isIonJS());
MOZ_ASSERT(!checkInvalidation());
return script()->ionScript();
}
const SafepointIndex* JSJitFrameIter::safepoint() const {
MOZ_ASSERT(isIonJS());
if (!cachedSafepointIndex_) {
cachedSafepointIndex_ =
ionScript()->getSafepointIndex(resumePCinCurrentFrame());
}
return cachedSafepointIndex_;
}
SnapshotOffset JSJitFrameIter::snapshotOffset() const {
MOZ_ASSERT(isIonScripted());
if (isBailoutJS()) {
return activation_->bailoutData()->snapshotOffset();
}
return osiIndex()->snapshotOffset();
}
const OsiIndex* JSJitFrameIter::osiIndex() const {
MOZ_ASSERT(isIonJS());
SafepointReader reader(ionScript(), safepoint());
return ionScript()->getOsiIndex(reader.osiReturnPointOffset());
}
bool JSJitFrameIter::isConstructing() const {
return CalleeTokenIsConstructing(calleeToken());
}
unsigned JSJitFrameIter::numActualArgs() const {
if (isScripted()) {
return jsFrame()->numActualArgs();
}
MOZ_ASSERT(isExitFrameLayout<NativeExitFrameLayout>());
return exitFrame()->as<NativeExitFrameLayout>()->argc();
}
void JSJitFrameIter::dumpBaseline() const {
MOZ_ASSERT(isBaselineJS());
fprintf(stderr, " JS Baseline frame\n");
if (isFunctionFrame()) {
fprintf(stderr, " callee fun: ");
#if defined(DEBUG) || defined(JS_JITSPEW)
DumpObject(callee());
#else
fprintf(stderr, "?\n");
#endif
} else {
fprintf(stderr, " global frame, no callee\n");
}
fprintf(stderr, " file %s line %u\n", script()->filename(),
script()->lineno());
JSContext* cx = TlsContext.get();
RootedScript script(cx);
jsbytecode* pc;
baselineScriptAndPc(script.address(), &pc);
fprintf(stderr, " script = %p, pc = %p (offset %u)\n", (void*)script, pc,
uint32_t(script->pcToOffset(pc)));
fprintf(stderr, " current op: %s\n", CodeName[*pc]);
fprintf(stderr, " actual args: %d\n", numActualArgs());
BaselineFrame* frame = baselineFrame();
for (unsigned i = 0; i < frame->numValueSlots(); i++) {
fprintf(stderr, " slot %u: ", i);
#if defined(DEBUG) || defined(JS_JITSPEW)
Value* v = frame->valueSlot(i);
DumpValue(*v);
#else
fprintf(stderr, "?\n");
#endif
}
}
void JSJitFrameIter::dump() const {
switch (type_) {
case FrameType::CppToJSJit:
fprintf(stderr, " Entry frame\n");
fprintf(stderr, " Frame size: %u\n",
unsigned(current()->prevFrameLocalSize()));
break;
case FrameType::BaselineJS:
dumpBaseline();
break;
case FrameType::BaselineStub:
fprintf(stderr, " Baseline stub frame\n");
fprintf(stderr, " Frame size: %u\n",
unsigned(current()->prevFrameLocalSize()));
break;
case FrameType::Bailout:
case FrameType::IonJS: {
InlineFrameIterator frames(TlsContext.get(), this);
for (;;) {
frames.dump();
if (!frames.more()) {
break;
}
++frames;
}
break;
}
case FrameType::Rectifier:
fprintf(stderr, " Rectifier frame\n");
fprintf(stderr, " Frame size: %u\n",
unsigned(current()->prevFrameLocalSize()));
break;
case FrameType::IonICCall:
fprintf(stderr, " Ion IC call\n");
fprintf(stderr, " Frame size: %u\n",
unsigned(current()->prevFrameLocalSize()));
break;
case FrameType::WasmToJSJit:
fprintf(stderr, " Fast wasm-to-JS entry frame\n");
fprintf(stderr, " Frame size: %u\n",
unsigned(current()->prevFrameLocalSize()));
break;
case FrameType::Exit:
fprintf(stderr, " Exit frame\n");
break;
case FrameType::JSJitToWasm:
fprintf(stderr, " Wasm exit frame\n");
break;
};
fputc('\n', stderr);
}
#ifdef DEBUG
bool JSJitFrameIter::verifyReturnAddressUsingNativeToBytecodeMap() {
MOZ_ASSERT(resumePCinCurrentFrame_ != nullptr);
if (type_ != FrameType::IonJS && type_ != FrameType::BaselineJS) {
return true;
}
JSRuntime* rt = TlsContext.get()->runtime();
if (!CurrentThreadCanAccessRuntime(rt)) {
return true;
}
if (!TlsContext.get()->isProfilerSamplingEnabled()) {
return true;
}
if (JS::RuntimeHeapIsMinorCollecting()) {
return true;
}
JitRuntime* jitrt = rt->jitRuntime();
const JitcodeGlobalEntry* entry =
jitrt->getJitcodeGlobalTable()->lookup(resumePCinCurrentFrame_);
if (!entry) {
return true;
}
JitSpew(JitSpew_Profiling, "Found nativeToBytecode entry for %p: %p - %p",
resumePCinCurrentFrame_, entry->nativeStartAddr(),
entry->nativeEndAddr());
JitcodeGlobalEntry::BytecodeLocationVector location;
uint32_t depth = UINT32_MAX;
if (!entry->callStackAtAddr(rt, resumePCinCurrentFrame_, location, &depth)) {
return false;
}
MOZ_ASSERT(depth > 0 && depth != UINT32_MAX);
MOZ_ASSERT(location.length() == depth);
JitSpew(JitSpew_Profiling, "Found bytecode location of depth %d:", depth);
for (size_t i = 0; i < location.length(); i++) {
JitSpew(JitSpew_Profiling, " %s:%u - %zu", location[i].script->filename(),
location[i].script->lineno(),
size_t(location[i].pc - location[i].script->code()));
}
if (type_ == FrameType::IonJS) {
InlineFrameIterator inlineFrames(TlsContext.get(), this);
for (size_t idx = 0; idx < location.length(); idx++) {
MOZ_ASSERT(idx < location.length());
MOZ_ASSERT_IF(idx < location.length() - 1, inlineFrames.more());
JitSpew(JitSpew_Profiling, "Match %d: ION %s:%u(%zu) vs N2B %s:%u(%zu)",
(int)idx, inlineFrames.script()->filename(),
inlineFrames.script()->lineno(),
size_t(inlineFrames.pc() - inlineFrames.script()->code()),
location[idx].script->filename(), location[idx].script->lineno(),
size_t(location[idx].pc - location[idx].script->code()));
MOZ_ASSERT(inlineFrames.script() == location[idx].script);
if (inlineFrames.more()) {
++inlineFrames;
}
}
}
return true;
}
#endif
JSJitProfilingFrameIterator::JSJitProfilingFrameIterator(JSContext* cx,
void* pc) {
if (!cx->profilingActivation()) {
type_ = FrameType::CppToJSJit;
fp_ = nullptr;
resumePCinCurrentFrame_ = nullptr;
return;
}
MOZ_ASSERT(cx->profilingActivation()->isJit());
JitActivation* act = cx->profilingActivation()->asJit();
if (!act->lastProfilingFrame()) {
type_ = FrameType::CppToJSJit;
fp_ = nullptr;
resumePCinCurrentFrame_ = nullptr;
return;
}
fp_ = (uint8_t*)act->lastProfilingFrame();
MOZ_ASSERT(cx->isProfilerSamplingEnabled());
if (tryInitWithPC(pc)) {
return;
}
JitcodeGlobalTable* table =
cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
if (tryInitWithTable(table, pc, false)) {
return;
}
void* lastCallSite = act->lastProfilingCallSite();
if (lastCallSite) {
if (tryInitWithPC(lastCallSite)) {
return;
}
if (tryInitWithTable(table, lastCallSite, true)) {
return;
}
}
MOZ_ASSERT(frameScript()->hasBaselineScript());
type_ = FrameType::BaselineJS;
resumePCinCurrentFrame_ = frameScript()->baselineScript()->method()->raw();
}
template <typename ReturnType = CommonFrameLayout*>
static inline ReturnType GetPreviousRawFrame(CommonFrameLayout* frame) {
size_t prevSize = frame->prevFrameLocalSize() + frame->headerSize();
return ReturnType((uint8_t*)frame + prevSize);
}
JSJitProfilingFrameIterator::JSJitProfilingFrameIterator(
CommonFrameLayout* fp) {
moveToNextFrame(fp);
}
bool JSJitProfilingFrameIterator::tryInitWithPC(void* pc) {
JSScript* callee = frameScript();
if (callee->hasIonScript() &&
callee->ionScript()->method()->containsNativePC(pc)) {
type_ = FrameType::IonJS;
resumePCinCurrentFrame_ = pc;
return true;
}
if (callee->hasBaselineScript() &&
callee->baselineScript()->method()->containsNativePC(pc)) {
type_ = FrameType::BaselineJS;
resumePCinCurrentFrame_ = pc;
return true;
}
return false;
}
bool JSJitProfilingFrameIterator::tryInitWithTable(JitcodeGlobalTable* table,
void* pc,
bool forLastCallSite) {
if (!pc) {
return false;
}
const JitcodeGlobalEntry* entry = table->lookup(pc);
if (!entry) {
return false;
}
JSScript* callee = frameScript();
MOZ_ASSERT(entry->isIon() || entry->isBaseline() || entry->isIonCache() ||
entry->isDummy());
if (entry->isDummy()) {
type_ = FrameType::CppToJSJit;
fp_ = nullptr;
resumePCinCurrentFrame_ = nullptr;
return true;
}
if (entry->isIon()) {
if (entry->ionEntry().getScript(0) != callee) {
return false;
}
type_ = FrameType::IonJS;
resumePCinCurrentFrame_ = pc;
return true;
}
if (entry->isBaseline()) {
if (forLastCallSite && entry->baselineEntry().script() != callee) {
return false;
}
type_ = FrameType::BaselineJS;
resumePCinCurrentFrame_ = pc;
return true;
}
if (entry->isIonCache()) {
void* ptr = entry->ionCacheEntry().rejoinAddr();
const JitcodeGlobalEntry& ionEntry = table->lookupInfallible(ptr);
MOZ_ASSERT(ionEntry.isIon());
if (ionEntry.ionEntry().getScript(0) != callee) {
return false;
}
type_ = FrameType::IonJS;
resumePCinCurrentFrame_ = pc;
return true;
}
return false;
}
void JSJitProfilingFrameIterator::fixBaselineReturnAddress() {
MOZ_ASSERT(type_ == FrameType::BaselineJS);
BaselineFrame* bl = (BaselineFrame*)(fp_ - BaselineFrame::FramePointerOffset -
BaselineFrame::Size());
if (BaselineDebugModeOSRInfo* info = bl->getDebugModeOSRInfo()) {
resumePCinCurrentFrame_ = info->resumeAddr;
return;
}
if (jsbytecode* override = bl->maybeOverridePc()) {
PCMappingSlotInfo slotInfo;
JSScript* script = bl->script();
resumePCinCurrentFrame_ =
script->baselineScript()->nativeCodeForPC(script, override, &slotInfo);
return;
}
}
void JSJitProfilingFrameIterator::operator++() {
JitFrameLayout* frame = framePtr();
moveToNextFrame(frame);
}
void JSJitProfilingFrameIterator::moveToWasmFrame(CommonFrameLayout* frame) {
resumePCinCurrentFrame_ = nullptr;
fp_ = GetPreviousRawFrame<uint8_t*>(frame);
type_ = FrameType::WasmToJSJit;
MOZ_ASSERT(!done());
}
void JSJitProfilingFrameIterator::moveToCppEntryFrame() {
resumePCinCurrentFrame_ = nullptr;
fp_ = nullptr;
type_ = FrameType::CppToJSJit;
}
void JSJitProfilingFrameIterator::moveToNextFrame(CommonFrameLayout* frame) {
FrameType prevType = frame->prevType();
if (prevType == FrameType::IonJS) {
resumePCinCurrentFrame_ = frame->returnAddress();
fp_ = GetPreviousRawFrame<uint8_t*>(frame);
type_ = FrameType::IonJS;
return;
}
if (prevType == FrameType::BaselineJS) {
resumePCinCurrentFrame_ = frame->returnAddress();
fp_ = GetPreviousRawFrame<uint8_t*>(frame);
type_ = FrameType::BaselineJS;
fixBaselineReturnAddress();
return;
}
if (prevType == FrameType::BaselineStub) {
BaselineStubFrameLayout* stubFrame =
GetPreviousRawFrame<BaselineStubFrameLayout*>(frame);
MOZ_ASSERT(stubFrame->prevType() == FrameType::BaselineJS);
resumePCinCurrentFrame_ = stubFrame->returnAddress();
fp_ = ((uint8_t*)stubFrame->reverseSavedFramePtr()) +
jit::BaselineFrame::FramePointerOffset;
type_ = FrameType::BaselineJS;
fixBaselineReturnAddress();
return;
}
if (prevType == FrameType::Rectifier) {
RectifierFrameLayout* rectFrame =
GetPreviousRawFrame<RectifierFrameLayout*>(frame);
FrameType rectPrevType = rectFrame->prevType();
if (rectPrevType == FrameType::IonJS) {
resumePCinCurrentFrame_ = rectFrame->returnAddress();
fp_ = GetPreviousRawFrame<uint8_t*>(rectFrame);
type_ = FrameType::IonJS;
return;
}
if (rectPrevType == FrameType::BaselineStub) {
BaselineStubFrameLayout* stubFrame =
GetPreviousRawFrame<BaselineStubFrameLayout*>(rectFrame);
resumePCinCurrentFrame_ = stubFrame->returnAddress();
fp_ = ((uint8_t*)stubFrame->reverseSavedFramePtr()) +
jit::BaselineFrame::FramePointerOffset;
type_ = FrameType::BaselineJS;
fixBaselineReturnAddress();
return;
}
if (rectPrevType == FrameType::WasmToJSJit) {
moveToWasmFrame(rectFrame);
return;
}
if (rectPrevType == FrameType::CppToJSJit) {
moveToCppEntryFrame();
return;
}
MOZ_CRASH("Bad frame type prior to rectifier frame.");
}
if (prevType == FrameType::IonICCall) {
IonICCallFrameLayout* callFrame =
GetPreviousRawFrame<IonICCallFrameLayout*>(frame);
MOZ_ASSERT(callFrame->prevType() == FrameType::IonJS);
resumePCinCurrentFrame_ = callFrame->returnAddress();
fp_ = GetPreviousRawFrame<uint8_t*>(callFrame);
type_ = FrameType::IonJS;
return;
}
if (prevType == FrameType::WasmToJSJit) {
moveToWasmFrame(frame);
return;
}
if (prevType == FrameType::CppToJSJit) {
moveToCppEntryFrame();
return;
}
MOZ_CRASH("Bad frame type.");
}