aboutsummaryrefslogtreecommitdiff
path: root/arch/powerpc/kvm/book3s_hv_rmhandlers.S
blob: b02f91e4c70dc5341bc450fa68f22b102ccd462e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 *
 * Derived from book3s_rmhandlers.S and other files, which are:
 *
 * Copyright SUSE Linux Products GmbH 2009
 *
 * Authors: Alexander Graf <agraf@suse.de>
 */

#include <asm/ppc_asm.h>
#include <asm/kvm_asm.h>
#include <asm/reg.h>
#include <asm/mmu.h>
#include <asm/page.h>
#include <asm/ptrace.h>
#include <asm/hvcall.h>
#include <asm/asm-offsets.h>
#include <asm/exception-64s.h>
#include <asm/kvm_book3s_asm.h>
#include <asm/mmu-hash64.h>

/*****************************************************************************
 *                                                                           *
 *        Real Mode handlers that need to be in the linear mapping           *
 *                                                                           *
 ****************************************************************************/

	.globl	kvmppc_skip_interrupt
kvmppc_skip_interrupt:
	mfspr	r13,SPRN_SRR0
	addi	r13,r13,4
	mtspr	SPRN_SRR0,r13
	GET_SCRATCH0(r13)
	rfid
	b	.

	.globl	kvmppc_skip_Hinterrupt
kvmppc_skip_Hinterrupt:
	mfspr	r13,SPRN_HSRR0
	addi	r13,r13,4
	mtspr	SPRN_HSRR0,r13
	GET_SCRATCH0(r13)
	hrfid
	b	.

/*
 * Call kvmppc_hv_entry in real mode.
 * Must be called with interrupts hard-disabled.
 *
 * Input Registers:
 *
 * LR = return address to continue at after eventually re-enabling MMU
 */
_GLOBAL(kvmppc_hv_entry_trampoline)
	mfmsr	r10
	LOAD_REG_ADDR(r5, kvmppc_hv_entry)
	li	r0,MSR_RI
	andc	r0,r10,r0
	li	r6,MSR_IR | MSR_DR
	andc	r6,r10,r6
	mtmsrd	r0,1		/* clear RI in MSR */
	mtsrr0	r5
	mtsrr1	r6
	RFI

/******************************************************************************
 *                                                                            *
 *                               Entry code                                   *
 *                                                                            *
 *****************************************************************************/

/*
 * We come in here when wakened from nap mode on a secondary hw thread.
 * Relocation is off and most register values are lost.
 * r13 points to the PACA.
 */
	.globl	kvm_start_guest
kvm_start_guest:
	ld	r1,PACAEMERGSP(r13)
	subi	r1,r1,STACK_FRAME_OVERHEAD
	ld	r2,PACATOC(r13)

	li	r0,KVM_HWTHREAD_IN_KVM
	stb	r0,HSTATE_HWTHREAD_STATE(r13)

	/* NV GPR values from power7_idle() will no longer be valid */
	li	r0,1
	stb	r0,PACA_NAPSTATELOST(r13)

	/* were we napping due to cede? */
	lbz	r0,HSTATE_NAPPING(r13)
	cmpwi	r0,0
	bne	kvm_end_cede

	/*
	 * We weren't napping due to cede, so this must be a secondary
	 * thread being woken up to run a guest, or being woken up due
	 * to a stray IPI.  (Or due to some machine check or hypervisor
	 * maintenance interrupt while the core is in KVM.)
	 */

	/* Check the wake reason in SRR1 to see why we got here */
	mfspr	r3,SPRN_SRR1
	rlwinm	r3,r3,44-31,0x7		/* extract wake reason field */
	cmpwi	r3,4			/* was it an external interrupt? */
	bne	27f			/* if not */
	ld	r5,HSTATE_XICS_PHYS(r13)
	li	r7,XICS_XIRR		/* if it was an external interrupt, */
	lwzcix	r8,r5,r7		/* get and ack the interrupt */
	sync
	clrldi.	r9,r8,40		/* get interrupt source ID. */
	beq	28f			/* none there? */
	cmpwi	r9,XICS_IPI		/* was it an IPI? */
	bne	29f
	li	r0,0xff
	li	r6,XICS_MFRR
	stbcix	r0,r5,r6		/* clear IPI */
	stwcix	r8,r5,r7		/* EOI the interrupt */
	sync				/* order loading of vcpu after that */

	/* get vcpu pointer, NULL if we have no vcpu to run */
	ld	r4,HSTATE_KVM_VCPU(r13)
	cmpdi	r4,0
	/* if we have no vcpu to run, go back to sleep */
	beq	kvm_no_guest
	b	kvmppc_hv_entry

27:	/* XXX should handle hypervisor maintenance interrupts etc. here */
	b	kvm_no_guest
28:	/* SRR1 said external but ICP said nope?? */
	b	kvm_no_guest
29:	/* External non-IPI interrupt to offline secondary thread? help?? */
	stw	r8,HSTATE_SAVED_XIRR(r13)
	b	kvm_no_guest

.global kvmppc_hv_entry
kvmppc_hv_entry:

	/* Required state:
	 *
	 * R4 = vcpu pointer
	 * MSR = ~IR|DR
	 * R13 = PACA
	 * R1 = host R1
	 * all other volatile GPRS = free
	 */
	mflr	r0
	std	r0, HSTATE_VMHANDLER(r13)

	/* Set partition DABR */
	/* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
	li	r5,3
	ld	r6,VCPU_DABR(r4)
	mtspr	SPRN_DABRX,r5
	mtspr	SPRN_DABR,r6
BEGIN_FTR_SECTION
	isync
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

	/* Load guest PMU registers */
	/* R4 is live here (vcpu pointer) */
	li	r3, 1
	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
	isync
	lwz	r3, VCPU_PMC(r4)	/* always load up guest PMU registers */
	lwz	r5, VCPU_PMC + 4(r4)	/* to prevent information leak */
	lwz	r6, VCPU_PMC + 8(r4)
	lwz	r7, VCPU_PMC + 12(r4)
	lwz	r8, VCPU_PMC + 16(r4)
	lwz	r9, VCPU_PMC + 20(r4)
BEGIN_FTR_SECTION
	lwz	r10, VCPU_PMC + 24(r4)
	lwz	r11, VCPU_PMC + 28(r4)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	mtspr	SPRN_PMC1, r3
	mtspr	SPRN_PMC2, r5
	mtspr	SPRN_PMC3, r6
	mtspr	SPRN_PMC4, r7
	mtspr	SPRN_PMC5, r8
	mtspr	SPRN_PMC6, r9
BEGIN_FTR_SECTION
	mtspr	SPRN_PMC7, r10
	mtspr	SPRN_PMC8, r11
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	ld	r3, VCPU_MMCR(r4)
	ld	r5, VCPU_MMCR + 8(r4)
	ld	r6, VCPU_MMCR + 16(r4)
	mtspr	SPRN_MMCR1, r5
	mtspr	SPRN_MMCRA, r6
	mtspr	SPRN_MMCR0, r3
	isync

	/* Load up FP, VMX and VSX registers */
	bl	kvmppc_load_fp

	ld	r14, VCPU_GPR(R14)(r4)
	ld	r15, VCPU_GPR(R15)(r4)
	ld	r16, VCPU_GPR(R16)(r4)
	ld	r17, VCPU_GPR(R17)(r4)
	ld	r18, VCPU_GPR(R18)(r4)
	ld	r19, VCPU_GPR(R19)(r4)
	ld	r20, VCPU_GPR(R20)(r4)
	ld	r21, VCPU_GPR(R21)(r4)
	ld	r22, VCPU_GPR(R22)(r4)
	ld	r23, VCPU_GPR(R23)(r4)
	ld	r24, VCPU_GPR(R24)(r4)
	ld	r25, VCPU_GPR(R25)(r4)
	ld	r26, VCPU_GPR(R26)(r4)
	ld	r27, VCPU_GPR(R27)(r4)
	ld	r28, VCPU_GPR(R28)(r4)
	ld	r29, VCPU_GPR(R29)(r4)
	ld	r30, VCPU_GPR(R30)(r4)
	ld	r31, VCPU_GPR(R31)(r4)

BEGIN_FTR_SECTION
	/* Switch DSCR to guest value */
	ld	r5, VCPU_DSCR(r4)
	mtspr	SPRN_DSCR, r5
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

	/*
	 * Set the decrementer to the guest decrementer.
	 */
	ld	r8,VCPU_DEC_EXPIRES(r4)
	mftb	r7
	subf	r3,r7,r8
	mtspr	SPRN_DEC,r3
	stw	r3,VCPU_DEC(r4)

	ld	r5, VCPU_SPRG0(r4)
	ld	r6, VCPU_SPRG1(r4)
	ld	r7, VCPU_SPRG2(r4)
	ld	r8, VCPU_SPRG3(r4)
	mtspr	SPRN_SPRG0, r5
	mtspr	SPRN_SPRG1, r6
	mtspr	SPRN_SPRG2, r7
	mtspr	SPRN_SPRG3, r8

	/* Save R1 in the PACA */
	std	r1, HSTATE_HOST_R1(r13)

	/* Increment yield count if they have a VPA */
	ld	r3, VCPU_VPA(r4)
	cmpdi	r3, 0
	beq	25f
	lwz	r5, LPPACA_YIELDCOUNT(r3)
	addi	r5, r5, 1
	stw	r5, LPPACA_YIELDCOUNT(r3)
	li	r6, 1
	stb	r6, VCPU_VPA_DIRTY(r4)
25:
	/* Load up DAR and DSISR */
	ld	r5, VCPU_DAR(r4)
	lwz	r6, VCPU_DSISR(r4)
	mtspr	SPRN_DAR, r5
	mtspr	SPRN_DSISR, r6

BEGIN_FTR_SECTION
	/* Restore AMR and UAMOR, set AMOR to all 1s */
	ld	r5,VCPU_AMR(r4)
	ld	r6,VCPU_UAMOR(r4)
	li	r7,-1
	mtspr	SPRN_AMR,r5
	mtspr	SPRN_UAMOR,r6
	mtspr	SPRN_AMOR,r7
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

	/* Clear out SLB */
	li	r6,0
	slbmte	r6,r6
	slbia
	ptesync

BEGIN_FTR_SECTION
	b	30f
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	/*
	 * POWER7 host -> guest partition switch code.
	 * We don't have to lock against concurrent tlbies,
	 * but we do have to coordinate across hardware threads.
	 */
	/* Increment entry count iff exit count is zero. */
	ld	r5,HSTATE_KVM_VCORE(r13)
	addi	r9,r5,VCORE_ENTRY_EXIT
21:	lwarx	r3,0,r9
	cmpwi	r3,0x100		/* any threads starting to exit? */
	bge	secondary_too_late	/* if so we're too late to the party */
	addi	r3,r3,1
	stwcx.	r3,0,r9
	bne	21b

	/* Primary thread switches to guest partition. */
	ld	r9,VCPU_KVM(r4)		/* pointer to struct kvm */
	lwz	r6,VCPU_PTID(r4)
	cmpwi	r6,0
	bne	20f
	ld	r6,KVM_SDR1(r9)
	lwz	r7,KVM_LPID(r9)
	li	r0,LPID_RSVD		/* switch to reserved LPID */
	mtspr	SPRN_LPID,r0
	ptesync
	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
	mtspr	SPRN_LPID,r7
	isync

	/* See if we need to flush the TLB */
	lhz	r6,PACAPACAINDEX(r13)	/* test_bit(cpu, need_tlb_flush) */
	clrldi	r7,r6,64-6		/* extract bit number (6 bits) */
	srdi	r6,r6,6			/* doubleword number */
	sldi	r6,r6,3			/* address offset */
	add	r6,r6,r9
	addi	r6,r6,KVM_NEED_FLUSH	/* dword in kvm->arch.need_tlb_flush */
	li	r0,1
	sld	r0,r0,r7
	ld	r7,0(r6)
	and.	r7,r7,r0
	beq	22f
23:	ldarx	r7,0,r6			/* if set, clear the bit */
	andc	r7,r7,r0
	stdcx.	r7,0,r6
	bne	23b
	li	r6,128			/* and flush the TLB */
	mtctr	r6
	li	r7,0x800		/* IS field = 0b10 */
	ptesync
28:	tlbiel	r7
	addi	r7,r7,0x1000
	bdnz	28b
	ptesync

22:	li	r0,1
	stb	r0,VCORE_IN_GUEST(r5)	/* signal secondaries to continue */
	b	10f

	/* Secondary threads wait for primary to have done partition switch */
20:	lbz	r0,VCORE_IN_GUEST(r5)
	cmpwi	r0,0
	beq	20b

	/* Set LPCR and RMOR. */
10:	ld	r8,KVM_LPCR(r9)
	mtspr	SPRN_LPCR,r8
	ld	r8,KVM_RMOR(r9)
	mtspr	SPRN_RMOR,r8
	isync

	/* Check if HDEC expires soon */
	mfspr	r3,SPRN_HDEC
	cmpwi	r3,10
	li	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
	mr	r9,r4
	blt	hdec_soon

	/* Save purr/spurr */
	mfspr	r5,SPRN_PURR
	mfspr	r6,SPRN_SPURR
	std	r5,HSTATE_PURR(r13)
	std	r6,HSTATE_SPURR(r13)
	ld	r7,VCPU_PURR(r4)
	ld	r8,VCPU_SPURR(r4)
	mtspr	SPRN_PURR,r7
	mtspr	SPRN_SPURR,r8
	b	31f

	/*
	 * PPC970 host -> guest partition switch code.
	 * We have to lock against concurrent tlbies,
	 * using native_tlbie_lock to lock against host tlbies
	 * and kvm->arch.tlbie_lock to lock against guest tlbies.
	 * We also have to invalidate the TLB since its
	 * entries aren't tagged with the LPID.
	 */
30:	ld	r9,VCPU_KVM(r4)		/* pointer to struct kvm */

	/* first take native_tlbie_lock */
	.section ".toc","aw"
toc_tlbie_lock:
	.tc	native_tlbie_lock[TC],native_tlbie_lock
	.previous
	ld	r3,toc_tlbie_lock@toc(2)
	lwz	r8,PACA_LOCK_TOKEN(r13)
24:	lwarx	r0,0,r3
	cmpwi	r0,0
	bne	24b
	stwcx.	r8,0,r3
	bne	24b
	isync

	ld	r7,KVM_LPCR(r9)		/* use kvm->arch.lpcr to store HID4 */
	li	r0,0x18f
	rotldi	r0,r0,HID4_LPID5_SH	/* all lpid bits in HID4 = 1 */
	or	r0,r7,r0
	ptesync
	sync
	mtspr	SPRN_HID4,r0		/* switch to reserved LPID */
	isync
	li	r0,0
	stw	r0,0(r3)		/* drop native_tlbie_lock */

	/* invalidate the whole TLB */
	li	r0,256
	mtctr	r0
	li	r6,0
25:	tlbiel	r6
	addi	r6,r6,0x1000
	bdnz	25b
	ptesync

	/* Take the guest's tlbie_lock */
	addi	r3,r9,KVM_TLBIE_LOCK
24:	lwarx	r0,0,r3
	cmpwi	r0,0
	bne	24b
	stwcx.	r8,0,r3
	bne	24b
	isync
	ld	r6,KVM_SDR1(r9)
	mtspr	SPRN_SDR1,r6		/* switch to partition page table */

	/* Set up HID4 with the guest's LPID etc. */
	sync
	mtspr	SPRN_HID4,r7
	isync

	/* drop the guest's tlbie_lock */
	li	r0,0
	stw	r0,0(r3)

	/* Check if HDEC expires soon */
	mfspr	r3,SPRN_HDEC
	cmpwi	r3,10
	li	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
	mr	r9,r4
	blt	hdec_soon

	/* Enable HDEC interrupts */
	mfspr	r0,SPRN_HID0
	li	r3,1
	rldimi	r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
	sync
	mtspr	SPRN_HID0,r0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0

	/* Load up guest SLB entries */
31:	lwz	r5,VCPU_SLB_MAX(r4)
	cmpwi	r5,0
	beq	9f
	mtctr	r5
	addi	r6,r4,VCPU_SLB
1:	ld	r8,VCPU_SLB_E(r6)
	ld	r9,VCPU_SLB_V(r6)
	slbmte	r9,r8
	addi	r6,r6,VCPU_SLB_SIZE
	bdnz	1b
9:

	/* Restore state of CTRL run bit; assume 1 on entry */
	lwz	r5,VCPU_CTRL(r4)
	andi.	r5,r5,1
	bne	4f
	mfspr	r6,SPRN_CTRLF
	clrrdi	r6,r6,1
	mtspr	SPRN_CTRLT,r6
4:
	ld	r6, VCPU_CTR(r4)
	lwz	r7, VCPU_XER(r4)

	mtctr	r6
	mtxer	r7

	ld	r10, VCPU_PC(r4)
	ld	r11, VCPU_MSR(r4)
kvmppc_cede_reentry:		/* r4 = vcpu, r13 = paca */
	ld	r6, VCPU_SRR0(r4)
	ld	r7, VCPU_SRR1(r4)

	/* r11 = vcpu->arch.msr & ~MSR_HV */
	rldicl	r11, r11, 63 - MSR_HV_LG, 1
	rotldi	r11, r11, 1 + MSR_HV_LG
	ori	r11, r11, MSR_ME

	/* Check if we can deliver an external or decrementer interrupt now */
	ld	r0,VCPU_PENDING_EXC(r4)
	lis	r8,(1 << BOOK3S_IRQPRIO_EXTERNAL_LEVEL)@h
	and	r0,r0,r8
	cmpdi	cr1,r0,0
	andi.	r0,r11,MSR_EE
	beq	cr1,11f
BEGIN_FTR_SECTION
	mfspr	r8,SPRN_LPCR
	ori	r8,r8,LPCR_MER
	mtspr	SPRN_LPCR,r8
	isync
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
	beq	5f
	li	r0,BOOK3S_INTERRUPT_EXTERNAL
12:	mr	r6,r10
	mr	r10,r0
	mr	r7,r11
	li	r11,(MSR_ME << 1) | 1	/* synthesize MSR_SF | MSR_ME */
	rotldi	r11,r11,63
	b	5f
11:	beq	5f
	mfspr	r0,SPRN_DEC
	cmpwi	r0,0
	li	r0,BOOK3S_INTERRUPT_DECREMENTER
	blt	12b

	/* Move SRR0 and SRR1 into the respective regs */
5:	mtspr	SPRN_SRR0, r6
	mtspr	SPRN_SRR1, r7

fast_guest_return:
	li	r0,0
	stb	r0,VCPU_CEDED(r4)	/* cancel cede */
	mtspr	SPRN_HSRR0,r10
	mtspr	SPRN_HSRR1,r11

	/* Activate guest mode, so faults get handled by KVM */
	li	r9, KVM_GUEST_MODE_GUEST
	stb	r9, HSTATE_IN_GUEST(r13)

	/* Enter guest */

BEGIN_FTR_SECTION
	ld	r5, VCPU_CFAR(r4)
	mtspr	SPRN_CFAR, r5
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)

	ld	r5, VCPU_LR(r4)
	lwz	r6, VCPU_CR(r4)
	mtlr	r5
	mtcr	r6

	ld	r0, VCPU_GPR(R0)(r4)
	ld	r1, VCPU_GPR(R1)(r4)
	ld	r2, VCPU_GPR(R2)(r4)
	ld	r3, VCPU_GPR(R3)(r4)
	ld	r5, VCPU_GPR(R5)(r4)
	ld	r6, VCPU_GPR(R6)(r4)
	ld	r7, VCPU_GPR(R7)(r4)
	ld	r8, VCPU_GPR(R8)(r4)
	ld	r9, VCPU_GPR(R9)(r4)
	ld	r10, VCPU_GPR(R10)(r4)
	ld	r11, VCPU_GPR(R11)(r4)
	ld	r12, VCPU_GPR(R12)(r4)
	ld	r13, VCPU_GPR(R13)(r4)

	ld	r4, VCPU_GPR(R4)(r4)

	hrfid
	b	.

/******************************************************************************
 *                                                                            *
 *                               Exit code                                    *
 *                                                                            *
 *****************************************************************************/

/*
 * We come here from the first-level interrupt handlers.
 */
	.globl	kvmppc_interrupt
kvmppc_interrupt:
	/*
	 * Register contents:
	 * R12		= interrupt vector
	 * R13		= PACA
	 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
	 * guest R13 saved in SPRN_SCRATCH0
	 */
	/* abuse host_r2 as third scratch area; we get r2 from PACATOC(r13) */
	std	r9, HSTATE_HOST_R2(r13)
	ld	r9, HSTATE_KVM_VCPU(r13)

	/* Save registers */

	std	r0, VCPU_GPR(R0)(r9)
	std	r1, VCPU_GPR(R1)(r9)
	std	r2, VCPU_GPR(R2)(r9)
	std	r3, VCPU_GPR(R3)(r9)
	std	r4, VCPU_GPR(R4)(r9)
	std	r5, VCPU_GPR(R5)(r9)
	std	r6, VCPU_GPR(R6)(r9)
	std	r7, VCPU_GPR(R7)(r9)
	std	r8, VCPU_GPR(R8)(r9)
	ld	r0, HSTATE_HOST_R2(r13)
	std	r0, VCPU_GPR(R9)(r9)
	std	r10, VCPU_GPR(R10)(r9)
	std	r11, VCPU_GPR(R11)(r9)
	ld	r3, HSTATE_SCRATCH0(r13)
	lwz	r4, HSTATE_SCRATCH1(r13)
	std	r3, VCPU_GPR(R12)(r9)
	stw	r4, VCPU_CR(r9)
BEGIN_FTR_SECTION
	ld	r3, HSTATE_CFAR(r13)
	std	r3, VCPU_CFAR(r9)
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)

	/* Restore R1/R2 so we can handle faults */
	ld	r1, HSTATE_HOST_R1(r13)
	ld	r2, PACATOC(r13)

	mfspr	r10, SPRN_SRR0
	mfspr	r11, SPRN_SRR1
	std	r10, VCPU_SRR0(r9)
	std	r11, VCPU_SRR1(r9)
	andi.	r0, r12, 2		/* need to read HSRR0/1? */
	beq	1f
	mfspr	r10, SPRN_HSRR0
	mfspr	r11, SPRN_HSRR1
	clrrdi	r12, r12, 2
1:	std	r10, VCPU_PC(r9)
	std	r11, VCPU_MSR(r9)

	GET_SCRATCH0(r3)
	mflr	r4
	std	r3, VCPU_GPR(R13)(r9)
	std	r4, VCPU_LR(r9)

	/* Unset guest mode */
	li	r0, KVM_GUEST_MODE_NONE
	stb	r0, HSTATE_IN_GUEST(r13)

	stw	r12,VCPU_TRAP(r9)

	/* Save HEIR (HV emulation assist reg) in last_inst
	   if this is an HEI (HV emulation interrupt, e40) */
	li	r3,KVM_INST_FETCH_FAILED
BEGIN_FTR_SECTION
	cmpwi	r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
	bne	11f
	mfspr	r3,SPRN_HEIR
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
11:	stw	r3,VCPU_LAST_INST(r9)

	/* these are volatile across C function calls */
	mfctr	r3
	mfxer	r4
	std	r3, VCPU_CTR(r9)
	stw	r4, VCPU_XER(r9)

BEGIN_FTR_SECTION
	/* If this is a page table miss then see if it's theirs or ours */
	cmpwi	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
	beq	kvmppc_hdsi
	cmpwi	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
	beq	kvmppc_hisi
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

	/* See if this is a leftover HDEC interrupt */
	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
	bne	2f
	mfspr	r3,SPRN_HDEC
	cmpwi	r3,0
	bge	ignore_hdec
2:
	/* See if this is an hcall we can handle in real mode */
	cmpwi	r12,BOOK3S_INTERRUPT_SYSCALL
	beq	hcall_try_real_mode

	/* Only handle external interrupts here on arch 206 and later */
BEGIN_FTR_SECTION
	b	ext_interrupt_to_host
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)

	/* External interrupt ? */
	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
	bne+	ext_interrupt_to_host

	/* External interrupt, first check for host_ipi. If this is
	 * set, we know the host wants us out so let's do it now
	 */
do_ext_interrupt:
	lbz	r0, HSTATE_HOST_IPI(r13)
	cmpwi	r0, 0
	bne	ext_interrupt_to_host

	/* Now read the interrupt from the ICP */
	ld	r5, HSTATE_XICS_PHYS(r13)
	li	r7, XICS_XIRR
	cmpdi	r5, 0
	beq-	ext_interrupt_to_host
	lwzcix	r3, r5, r7
	rlwinm.	r0, r3, 0, 0xffffff
	sync
	beq	3f		/* if nothing pending in the ICP */

	/* We found something in the ICP...
	 *
	 * If it's not an IPI, stash it in the PACA and return to
	 * the host, we don't (yet) handle directing real external
	 * interrupts directly to the guest
	 */
	cmpwi	r0, XICS_IPI
	bne	ext_stash_for_host

	/* It's an IPI, clear the MFRR and EOI it */
	li	r0, 0xff
	li	r6, XICS_MFRR
	stbcix	r0, r5, r6		/* clear the IPI */
	stwcix	r3, r5, r7		/* EOI it */
	sync

	/* We need to re-check host IPI now in case it got set in the
	 * meantime. If it's clear, we bounce the interrupt to the
	 * guest
	 */
	lbz	r0, HSTATE_HOST_IPI(r13)
	cmpwi	r0, 0
	bne-	1f

	/* Allright, looks like an IPI for the guest, we need to set MER */
3:
	/* Check if any CPU is heading out to the host, if so head out too */
	ld	r5, HSTATE_KVM_VCORE(r13)
	lwz	r0, VCORE_ENTRY_EXIT(r5)
	cmpwi	r0, 0x100
	bge	ext_interrupt_to_host

	/* See if there is a pending interrupt for the guest */
	mfspr	r8, SPRN_LPCR
	ld	r0, VCPU_PENDING_EXC(r9)
	/* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
	rldicl.	r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
	rldimi	r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
	beq	2f

	/* And if the guest EE is set, we can deliver immediately, else
	 * we return to the guest with MER set
	 */
	andi.	r0, r11, MSR_EE
	beq	2f
	mtspr	SPRN_SRR0, r10
	mtspr	SPRN_SRR1, r11
	li	r10, BOOK3S_INTERRUPT_EXTERNAL
	li	r11, (MSR_ME << 1) | 1	/* synthesize MSR_SF | MSR_ME */
	rotldi	r11, r11, 63
2:	mr	r4, r9
	mtspr	SPRN_LPCR, r8
	b	fast_guest_return

	/* We raced with the host, we need to resend that IPI, bummer */
1:	li	r0, IPI_PRIORITY
	stbcix	r0, r5, r6		/* set the IPI */
	sync
	b	ext_interrupt_to_host

ext_stash_for_host:
	/* It's not an IPI and it's for the host, stash it in the PACA
	 * before exit, it will be picked up by the host ICP driver
	 */
	stw	r3, HSTATE_SAVED_XIRR(r13)
ext_interrupt_to_host:

guest_exit_cont:		/* r9 = vcpu, r12 = trap, r13 = paca */
	/* Save DEC */
	mfspr	r5,SPRN_DEC
	mftb	r6
	extsw	r5,r5
	add	r5,r5,r6
	std	r5,VCPU_DEC_EXPIRES(r9)

	/* Save more register state  */
	mfdar	r6
	mfdsisr	r7
	std	r6, VCPU_DAR(r9)
	stw	r7, VCPU_DSISR(r9)
BEGIN_FTR_SECTION
	/* don't overwrite fault_dar/fault_dsisr if HDSI */
	cmpwi	r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
	beq	6f
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
	std	r6, VCPU_FAULT_DAR(r9)
	stw	r7, VCPU_FAULT_DSISR(r9)

	/* See if it is a machine check */
	cmpwi	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
	beq	machine_check_realmode
mc_cont:

	/* Save guest CTRL register, set runlatch to 1 */
6:	mfspr	r6,SPRN_CTRLF
	stw	r6,VCPU_CTRL(r9)
	andi.	r0,r6,1
	bne	4f
	ori	r6,r6,1
	mtspr	SPRN_CTRLT,r6
4:
	/* Read the guest SLB and save it away */
	lwz	r0,VCPU_SLB_NR(r9)	/* number of entries in SLB */
	mtctr	r0
	li	r6,0
	addi	r7,r9,VCPU_SLB
	li	r5,0
1:	slbmfee	r8,r6
	andis.	r0,r8,SLB_ESID_V@h
	beq	2f
	add	r8,r8,r6		/* put index in */
	slbmfev	r3,r6
	std	r8,VCPU_SLB_E(r7)
	std	r3,VCPU_SLB_V(r7)
	addi	r7,r7,VCPU_SLB_SIZE
	addi	r5,r5,1
2:	addi	r6,r6,1
	bdnz	1b
	stw	r5,VCPU_SLB_MAX(r9)

	/*
	 * Save the guest PURR/SPURR
	 */
BEGIN_FTR_SECTION
	mfspr	r5,SPRN_PURR
	mfspr	r6,SPRN_SPURR
	ld	r7,VCPU_PURR(r9)
	ld	r8,VCPU_SPURR(r9)
	std	r5,VCPU_PURR(r9)
	std	r6,VCPU_SPURR(r9)
	subf	r5,r7,r5
	subf	r6,r8,r6

	/*
	 * Restore host PURR/SPURR and add guest times
	 * so that the time in the guest gets accounted.
	 */
	ld	r3,HSTATE_PURR(r13)
	ld	r4,HSTATE_SPURR(r13)
	add	r3,r3,r5
	add	r4,r4,r6
	mtspr	SPRN_PURR,r3
	mtspr	SPRN_SPURR,r4
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_201)

	/* Clear out SLB */
	li	r5,0
	slbmte	r5,r5
	slbia
	ptesync

hdec_soon:			/* r9 = vcpu, r12 = trap, r13 = paca */
BEGIN_FTR_SECTION
	b	32f
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	/*
	 * POWER7 guest -> host partition switch code.
	 * We don't have to lock against tlbies but we do
	 * have to coordinate the hardware threads.
	 */
	/* Increment the threads-exiting-guest count in the 0xff00
	   bits of vcore->entry_exit_count */
	lwsync
	ld	r5,HSTATE_KVM_VCORE(r13)
	addi	r6,r5,VCORE_ENTRY_EXIT
41:	lwarx	r3,0,r6
	addi	r0,r3,0x100
	stwcx.	r0,0,r6
	bne	41b
	lwsync

	/*
	 * At this point we have an interrupt that we have to pass
	 * up to the kernel or qemu; we can't handle it in real mode.
	 * Thus we have to do a partition switch, so we have to
	 * collect the other threads, if we are the first thread
	 * to take an interrupt.  To do this, we set the HDEC to 0,
	 * which causes an HDEC interrupt in all threads within 2ns
	 * because the HDEC register is shared between all 4 threads.
	 * However, we don't need to bother if this is an HDEC
	 * interrupt, since the other threads will already be on their
	 * way here in that case.
	 */
	cmpwi	r3,0x100	/* Are we the first here? */
	bge	43f
	cmpwi	r3,1		/* Are any other threads in the guest? */
	ble	43f
	cmpwi	r12,BOOK3S_INTERRUPT_HV_DECREMENTER
	beq	40f
	li	r0,0
	mtspr	SPRN_HDEC,r0
40:
	/*
	 * Send an IPI to any napping threads, since an HDEC interrupt
	 * doesn't wake CPUs up from nap.
	 */
	lwz	r3,VCORE_NAPPING_THREADS(r5)
	lwz	r4,VCPU_PTID(r9)
	li	r0,1
	sld	r0,r0,r4
	andc.	r3,r3,r0		/* no sense IPI'ing ourselves */
	beq	43f
	mulli	r4,r4,PACA_SIZE		/* get paca for thread 0 */
	subf	r6,r4,r13
42:	andi.	r0,r3,1
	beq	44f
	ld	r8,HSTATE_XICS_PHYS(r6)	/* get thread's XICS reg addr */
	li	r0,IPI_PRIORITY
	li	r7,XICS_MFRR
	stbcix	r0,r7,r8		/* trigger the IPI */
44:	srdi.	r3,r3,1
	addi	r6,r6,PACA_SIZE
	bne	42b

	/* Secondary threads wait for primary to do partition switch */
43:	ld	r4,VCPU_KVM(r9)		/* pointer to struct kvm */
	ld	r5,HSTATE_KVM_VCORE(r13)
	lwz	r3,VCPU_PTID(r9)
	cmpwi	r3,0
	beq	15f
	HMT_LOW
13:	lbz	r3,VCORE_IN_GUEST(r5)
	cmpwi	r3,0
	bne	13b
	HMT_MEDIUM
	b	16f

	/* Primary thread waits for all the secondaries to exit guest */
15:	lwz	r3,VCORE_ENTRY_EXIT(r5)
	srwi	r0,r3,8
	clrldi	r3,r3,56
	cmpw	r3,r0
	bne	15b
	isync

	/* Primary thread switches back to host partition */
	ld	r6,KVM_HOST_SDR1(r4)
	lwz	r7,KVM_HOST_LPID(r4)
	li	r8,LPID_RSVD		/* switch to reserved LPID */
	mtspr	SPRN_LPID,r8
	ptesync
	mtspr	SPRN_SDR1,r6		/* switch to partition page table */
	mtspr	SPRN_LPID,r7
	isync
	li	r0,0
	stb	r0,VCORE_IN_GUEST(r5)
	lis	r8,0x7fff		/* MAX_INT@h */
	mtspr	SPRN_HDEC,r8

16:	ld	r8,KVM_HOST_LPCR(r4)
	mtspr	SPRN_LPCR,r8
	isync
	b	33f

	/*
	 * PPC970 guest -> host partition switch code.
	 * We have to lock against concurrent tlbies, and
	 * we have to flush the whole TLB.
	 */
32:	ld	r4,VCPU_KVM(r9)		/* pointer to struct kvm */

	/* Take the guest's tlbie_lock */
	lwz	r8,PACA_LOCK_TOKEN(r13)
	addi	r3,r4,KVM_TLBIE_LOCK
24:	lwarx	r0,0,r3
	cmpwi	r0,0
	bne	24b
	stwcx.	r8,0,r3
	bne	24b
	isync

	ld	r7,KVM_HOST_LPCR(r4)	/* use kvm->arch.host_lpcr for HID4 */
	li	r0,0x18f
	rotldi	r0,r0,HID4_LPID5_SH	/* all lpid bits in HID4 = 1 */
	or	r0,r7,r0
	ptesync
	sync
	mtspr	SPRN_HID4,r0		/* switch to reserved LPID */
	isync
	li	r0,0
	stw	r0,0(r3)		/* drop guest tlbie_lock */

	/* invalidate the whole TLB */
	li	r0,256
	mtctr	r0
	li	r6,0
25:	tlbiel	r6
	addi	r6,r6,0x1000
	bdnz	25b
	ptesync

	/* take native_tlbie_lock */
	ld	r3,toc_tlbie_lock@toc(2)
24:	lwarx	r0,0,r3
	cmpwi	r0,0
	bne	24b
	stwcx.	r8,0,r3
	bne	24b
	isync

	ld	r6,KVM_HOST_SDR1(r4)
	mtspr	SPRN_SDR1,r6		/* switch to host page table */

	/* Set up host HID4 value */
	sync
	mtspr	SPRN_HID4,r7
	isync
	li	r0,0
	stw	r0,0(r3)		/* drop native_tlbie_lock */

	lis	r8,0x7fff		/* MAX_INT@h */
	mtspr	SPRN_HDEC,r8

	/* Disable HDEC interrupts */
	mfspr	r0,SPRN_HID0
	li	r3,0
	rldimi	r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
	sync
	mtspr	SPRN_HID0,r0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0
	mfspr	r0,SPRN_HID0

	/* load host SLB entries */
33:	ld	r8,PACA_SLBSHADOWPTR(r13)

	.rept	SLB_NUM_BOLTED
	ld	r5,SLBSHADOW_SAVEAREA(r8)
	ld	r6,SLBSHADOW_SAVEAREA+8(r8)
	andis.	r7,r5,SLB_ESID_V@h
	beq	1f
	slbmte	r6,r5
1:	addi	r8,r8,16
	.endr

	/* Save and reset AMR and UAMOR before turning on the MMU */
BEGIN_FTR_SECTION
	mfspr	r5,SPRN_AMR
	mfspr	r6,SPRN_UAMOR
	std	r5,VCPU_AMR(r9)
	std	r6,VCPU_UAMOR(r9)
	li	r6,0
	mtspr	SPRN_AMR,r6
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

	/* Switch DSCR back to host value */
BEGIN_FTR_SECTION
	mfspr	r8, SPRN_DSCR
	ld	r7, HSTATE_DSCR(r13)
	std	r8, VCPU_DSCR(r7)
	mtspr	SPRN_DSCR, r7
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

	/* Save non-volatile GPRs */
	std	r14, VCPU_GPR(R14)(r9)
	std	r15, VCPU_GPR(R15)(r9)
	std	r16, VCPU_GPR(R16)(r9)
	std	r17, VCPU_GPR(R17)(r9)
	std	r18, VCPU_GPR(R18)(r9)
	std	r19, VCPU_GPR(R19)(r9)
	std	r20, VCPU_GPR(R20)(r9)
	std	r21, VCPU_GPR(R21)(r9)
	std	r22, VCPU_GPR(R22)(r9)
	std	r23, VCPU_GPR(R23)(r9)
	std	r24, VCPU_GPR(R24)(r9)
	std	r25, VCPU_GPR(R25)(r9)
	std	r26, VCPU_GPR(R26)(r9)
	std	r27, VCPU_GPR(R27)(r9)
	std	r28, VCPU_GPR(R28)(r9)
	std	r29, VCPU_GPR(R29)(r9)
	std	r30, VCPU_GPR(R30)(r9)
	std	r31, VCPU_GPR(R31)(r9)

	/* Save SPRGs */
	mfspr	r3, SPRN_SPRG0
	mfspr	r4, SPRN_SPRG1
	mfspr	r5, SPRN_SPRG2
	mfspr	r6, SPRN_SPRG3
	std	r3, VCPU_SPRG0(r9)
	std	r4, VCPU_SPRG1(r9)
	std	r5, VCPU_SPRG2(r9)
	std	r6, VCPU_SPRG3(r9)

	/* save FP state */
	mr	r3, r9
	bl	.kvmppc_save_fp

	/* Increment yield count if they have a VPA */
	ld	r8, VCPU_VPA(r9)	/* do they have a VPA? */
	cmpdi	r8, 0
	beq	25f
	lwz	r3, LPPACA_YIELDCOUNT(r8)
	addi	r3, r3, 1
	stw	r3, LPPACA_YIELDCOUNT(r8)
	li	r3, 1
	stb	r3, VCPU_VPA_DIRTY(r9)
25:
	/* Save PMU registers if requested */
	/* r8 and cr0.eq are live here */
	li	r3, 1
	sldi	r3, r3, 31		/* MMCR0_FC (freeze counters) bit */
	mfspr	r4, SPRN_MMCR0		/* save MMCR0 */
	mtspr	SPRN_MMCR0, r3		/* freeze all counters, disable ints */
	mfspr	r6, SPRN_MMCRA
BEGIN_FTR_SECTION
	/* On P7, clear MMCRA in order to disable SDAR updates */
	li	r7, 0
	mtspr	SPRN_MMCRA, r7
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
	isync
	beq	21f			/* if no VPA, save PMU stuff anyway */
	lbz	r7, LPPACA_PMCINUSE(r8)
	cmpwi	r7, 0			/* did they ask for PMU stuff to be saved? */
	bne	21f
	std	r3, VCPU_MMCR(r9)	/* if not, set saved MMCR0 to FC */
	b	22f
21:	mfspr	r5, SPRN_MMCR1
	std	r4, VCPU_MMCR(r9)
	std	r5, VCPU_MMCR + 8(r9)
	std	r6, VCPU_MMCR + 16(r9)
	mfspr	r3, SPRN_PMC1
	mfspr	r4, SPRN_PMC2
	mfspr	r5, SPRN_PMC3
	mfspr	r6, SPRN_PMC4
	mfspr	r7, SPRN_PMC5
	mfspr	r8, SPRN_PMC6
BEGIN_FTR_SECTION
	mfspr	r10, SPRN_PMC7
	mfspr	r11, SPRN_PMC8
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	stw	r3, VCPU_PMC(r9)
	stw	r4, VCPU_PMC + 4(r9)
	stw	r5, VCPU_PMC + 8(r9)
	stw	r6, VCPU_PMC + 12(r9)
	stw	r7, VCPU_PMC + 16(r9)
	stw	r8, VCPU_PMC + 20(r9)
BEGIN_FTR_SECTION
	stw	r10, VCPU_PMC + 24(r9)
	stw	r11, VCPU_PMC + 28(r9)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
22:

	/* Secondary threads go off to take a nap on POWER7 */
BEGIN_FTR_SECTION
	lwz	r0,VCPU_PTID(r9)
	cmpwi	r0,0
	bne	secondary_nap
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

	/* Restore host DABR and DABRX */
	ld	r5,HSTATE_DABR(r13)
	li	r6,7
	mtspr	SPRN_DABR,r5
	mtspr	SPRN_DABRX,r6

	/* Restore SPRG3 */
	ld	r3,PACA_SPRG3(r13)
	mtspr	SPRN_SPRG3,r3

	/*
	 * Reload DEC.  HDEC interrupts were disabled when
	 * we reloaded the host's LPCR value.
	 */
	ld	r3, HSTATE_DECEXP(r13)
	mftb	r4
	subf	r4, r4, r3
	mtspr	SPRN_DEC, r4

	/* Reload the host's PMU registers */
	ld	r3, PACALPPACAPTR(r13)	/* is the host using the PMU? */
	lbz	r4, LPPACA_PMCINUSE(r3)
	cmpwi	r4, 0
	beq	23f			/* skip if not */
	lwz	r3, HSTATE_PMC(r13)
	lwz	r4, HSTATE_PMC + 4(r13)
	lwz	r5, HSTATE_PMC + 8(r13)
	lwz	r6, HSTATE_PMC + 12(r13)
	lwz	r8, HSTATE_PMC + 16(r13)
	lwz	r9, HSTATE_PMC + 20(r13)
BEGIN_FTR_SECTION
	lwz	r10, HSTATE_PMC + 24(r13)
	lwz	r11, HSTATE_PMC + 28(r13)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	mtspr	SPRN_PMC1, r3
	mtspr	SPRN_PMC2, r4
	mtspr	SPRN_PMC3, r5
	mtspr	SPRN_PMC4, r6
	mtspr	SPRN_PMC5, r8
	mtspr	SPRN_PMC6, r9
BEGIN_FTR_SECTION
	mtspr	SPRN_PMC7, r10
	mtspr	SPRN_PMC8, r11
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
	ld	r3, HSTATE_MMCR(r13)
	ld	r4, HSTATE_MMCR + 8(r13)
	ld	r5, HSTATE_MMCR + 16(r13)
	mtspr	SPRN_MMCR1, r4
	mtspr	SPRN_MMCRA, r5
	mtspr	SPRN_MMCR0, r3
	isync
23:
	/*
	 * For external and machine check interrupts, we need
	 * to call the Linux handler to process the interrupt.
	 * We do that by jumping to absolute address 0x500 for
	 * external interrupts, or the machine_check_fwnmi label
	 * for machine checks (since firmware might have patched
	 * the vector area at 0x200).  The [h]rfid at the end of the
	 * handler will return to the book3s_hv_interrupts.S code.
	 * For other interrupts we do the rfid to get back
	 * to the book3s_hv_interrupts.S code here.
	 */
	ld	r8, HSTATE_VMHANDLER(r13)
	ld	r7, HSTATE_HOST_MSR(r13)

	cmpwi	cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
	cmpwi	r12, BOOK3S_INTERRUPT_EXTERNAL
BEGIN_FTR_SECTION
	beq	11f
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)

	/* RFI into the highmem handler, or branch to interrupt handler */
	mfmsr	r6
	li	r0, MSR_RI
	andc	r6, r6, r0
	mtmsrd	r6, 1			/* Clear RI in MSR */
	mtsrr0	r8
	mtsrr1	r7
	beqa	0x500			/* external interrupt (PPC970) */
	beq	cr1, 13f		/* machine check */
	RFI

	/* On POWER7, we have external interrupts set to use HSRR0/1 */
11:	mtspr	SPRN_HSRR0, r8
	mtspr	SPRN_HSRR1, r7
	ba	0x500

13:	b	machine_check_fwnmi

/*
 * Check whether an HDSI is an HPTE not found fault or something else.
 * If it is an HPTE not found fault that is due to the guest accessing
 * a page that they have mapped but which we have paged out, then
 * we continue on with the guest exit path.  In all other cases,
 * reflect the HDSI to the guest as a DSI.
 */
kvmppc_hdsi:
	mfspr	r4, SPRN_HDAR
	mfspr	r6, SPRN_HDSISR
	/* HPTE not found fault or protection fault? */
	andis.	r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
	beq	1f			/* if not, send it to the guest */
	andi.	r0, r11, MSR_DR		/* data relocation enabled? */
	beq	3f
	clrrdi	r0, r4, 28
	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
	bne	1f			/* if no SLB entry found */
4:	std	r4, VCPU_FAULT_DAR(r9)
	stw	r6, VCPU_FAULT_DSISR(r9)

	/* Search the hash table. */
	mr	r3, r9			/* vcpu pointer */
	li	r7, 1			/* data fault */
	bl	.kvmppc_hpte_hv_fault
	ld	r9, HSTATE_KVM_VCPU(r13)
	ld	r10, VCPU_PC(r9)
	ld	r11, VCPU_MSR(r9)
	li	r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
	cmpdi	r3, 0			/* retry the instruction */
	beq	6f
	cmpdi	r3, -1			/* handle in kernel mode */
	beq	guest_exit_cont
	cmpdi	r3, -2			/* MMIO emulation; need instr word */
	beq	2f

	/* Synthesize a DSI for the guest */
	ld	r4, VCPU_FAULT_DAR(r9)
	mr	r6, r3
1:	mtspr	SPRN_DAR, r4
	mtspr	SPRN_DSISR, r6
	mtspr	SPRN_SRR0, r10
	mtspr	SPRN_SRR1, r11
	li	r10, BOOK3S_INTERRUPT_DATA_STORAGE
	li	r11, (MSR_ME << 1) | 1	/* synthesize MSR_SF | MSR_ME */
	rotldi	r11, r11, 63
fast_interrupt_c_return:
6:	ld	r7, VCPU_CTR(r9)
	lwz	r8, VCPU_XER(r9)
	mtctr	r7
	mtxer	r8
	mr	r4, r9
	b	fast_guest_return

3:	ld	r5, VCPU_KVM(r9)	/* not relocated, use VRMA */
	ld	r5, KVM_VRMA_SLB_V(r5)
	b	4b

	/* If this is for emulated MMIO, load the instruction word */
2:	li	r8, KVM_INST_FETCH_FAILED	/* In case lwz faults */

	/* Set guest mode to 'jump over instruction' so if lwz faults
	 * we'll just continue at the next IP. */
	li	r0, KVM_GUEST_MODE_SKIP
	stb	r0, HSTATE_IN_GUEST(r13)

	/* Do the access with MSR:DR enabled */
	mfmsr	r3
	ori	r4, r3, MSR_DR		/* Enable paging for data */
	mtmsrd	r4
	lwz	r8, 0(r10)
	mtmsrd	r3

	/* Store the result */
	stw	r8, VCPU_LAST_INST(r9)

	/* Unset guest mode. */
	li	r0, KVM_GUEST_MODE_NONE
	stb	r0, HSTATE_IN_GUEST(r13)
	b	guest_exit_cont

/*
 * Similarly for an HISI, reflect it to the guest as an ISI unless
 * it is an HPTE not found fault for a page that we have paged out.
 */
kvmppc_hisi:
	andis.	r0, r11, SRR1_ISI_NOPT@h
	beq	1f
	andi.	r0, r11, MSR_IR		/* instruction relocation enabled? */
	beq	3f
	clrrdi	r0, r10, 28
	PPC_SLBFEE_DOT(R5, R0)		/* if so, look up SLB */
	bne	1f			/* if no SLB entry found */
4:
	/* Search the hash table. */
	mr	r3, r9			/* vcpu pointer */
	mr	r4, r10
	mr	r6, r11
	li	r7, 0			/* instruction fault */
	bl	.kvmppc_hpte_hv_fault
	ld	r9, HSTATE_KVM_VCPU(r13)
	ld	r10, VCPU_PC(r9)
	ld	r11, VCPU_MSR(r9)
	li	r12, BOOK3S_INTERRUPT_H_INST_STORAGE
	cmpdi	r3, 0			/* retry the instruction */
	beq	fast_interrupt_c_return
	cmpdi	r3, -1			/* handle in kernel mode */
	beq	guest_exit_cont

	/* Synthesize an ISI for the guest */
	mr	r11, r3
1:	mtspr	SPRN_SRR0, r10
	mtspr	SPRN_SRR1, r11
	li	r10, BOOK3S_INTERRUPT_INST_STORAGE
	li	r11, (MSR_ME << 1) | 1	/* synthesize MSR_SF | MSR_ME */
	rotldi	r11, r11, 63
	b	fast_interrupt_c_return

3:	ld	r6, VCPU_KVM(r9)	/* not relocated, use VRMA */
	ld	r5, KVM_VRMA_SLB_V(r6)
	b	4b

/*
 * Try to handle an hcall in real mode.
 * Returns to the guest if we handle it, or continues on up to
 * the kernel if we can't (i.e. if we don't have a handler for
 * it, or if the handler returns H_TOO_HARD).
 */
	.globl	hcall_try_real_mode
hcall_try_real_mode:
	ld	r3,VCPU_GPR(R3)(r9)
	andi.	r0,r11,MSR_PR
	bne	guest_exit_cont
	clrrdi	r3,r3,2
	cmpldi	r3,hcall_real_table_end - hcall_real_table
	bge	guest_exit_cont
	LOAD_REG_ADDR(r4, hcall_real_table)
	lwzx	r3,r3,r4
	cmpwi	r3,0
	beq	guest_exit_cont
	add	r3,r3,r4
	mtctr	r3
	mr	r3,r9		/* get vcpu pointer */
	ld	r4,VCPU_GPR(R4)(r9)
	bctrl
	cmpdi	r3,H_TOO_HARD
	beq	hcall_real_fallback
	ld	r4,HSTATE_KVM_VCPU(r13)
	std	r3,VCPU_GPR(R3)(r4)
	ld	r10,VCPU_PC(r4)
	ld	r11,VCPU_MSR(r4)
	b	fast_guest_return

	/* We've attempted a real mode hcall, but it's punted it back
	 * to userspace.  We need to restore some clobbered volatiles
	 * before resuming the pass-it-to-qemu path */
hcall_real_fallback:
	li	r12,BOOK3S_INTERRUPT_SYSCALL
	ld	r9, HSTATE_KVM_VCPU(r13)

	b	guest_exit_cont

	.globl	hcall_real_table
hcall_real_table:
	.long	0		/* 0 - unused */
	.long	.kvmppc_h_remove - hcall_real_table
	.long	.kvmppc_h_enter - hcall_real_table
	.long	.kvmppc_h_read - hcall_real_table
	.long	0		/* 0x10 - H_CLEAR_MOD */
	.long	0		/* 0x14 - H_CLEAR_REF */
	.long	.kvmppc_h_protect - hcall_real_table
	.long	0		/* 0x1c - H_GET_TCE */
	.long	.kvmppc_h_put_tce - hcall_real_table
	.long	0		/* 0x24 - H_SET_SPRG0 */
	.long	.kvmppc_h_set_dabr - hcall_real_table
	.long	0		/* 0x2c */
	.long	0		/* 0x30 */
	.long	0		/* 0x34 */
	.long	0		/* 0x38 */
	.long	0		/* 0x3c */
	.long	0		/* 0x40 */
	.long	0		/* 0x44 */
	.long	0		/* 0x48 */
	.long	0		/* 0x4c */
	.long	0		/* 0x50 */
	.long	0		/* 0x54 */
	.long	0		/* 0x58 */
	.long	0		/* 0x5c */
	.long	0		/* 0x60 */
#ifdef CONFIG_KVM_XICS
	.long	.kvmppc_rm_h_eoi - hcall_real_table
	.long	.kvmppc_rm_h_cppr - hcall_real_table
	.long	.kvmppc_rm_h_ipi - hcall_real_table
	.long	0		/* 0x70 - H_IPOLL */
	.long	.kvmppc_rm_h_xirr - hcall_real_table
#else
	.long	0		/* 0x64 - H_EOI */
	.long	0		/* 0x68 - H_CPPR */
	.long	0		/* 0x6c - H_IPI */
	.long	0		/* 0x70 - H_IPOLL */
	.long	0		/* 0x74 - H_XIRR */
#endif
	.long	0		/* 0x78 */
	.long	0		/* 0x7c */
	.long	0		/* 0x80 */
	.long	0		/* 0x84 */
	.long	0		/* 0x88 */
	.long	0		/* 0x8c */
	.long	0		/* 0x90 */
	.long	0		/* 0x94 */
	.long	0		/* 0x98 */
	.long	0		/* 0x9c */
	.long	0		/* 0xa0 */
	.long	0		/* 0xa4 */
	.long	0		/* 0xa8 */
	.long	0		/* 0xac */
	.long	0		/* 0xb0 */
	.long	0		/* 0xb4 */
	.long	0		/* 0xb8 */
	.long	0		/* 0xbc */
	.long	0		/* 0xc0 */
	.long	0		/* 0xc4 */
	.long	0		/* 0xc8 */
	.long	0		/* 0xcc */
	.long	0		/* 0xd0 */
	.long	0		/* 0xd4 */
	.long	0		/* 0xd8 */
	.long	0		/* 0xdc */
	.long	.kvmppc_h_cede - hcall_real_table
	.long	0		/* 0xe4 */
	.long	0		/* 0xe8 */
	.long	0		/* 0xec */
	.long	0		/* 0xf0 */
	.long	0		/* 0xf4 */
	.long	0		/* 0xf8 */
	.long	0		/* 0xfc */
	.long	0		/* 0x100 */
	.long	0		/* 0x104 */
	.long	0		/* 0x108 */
	.long	0		/* 0x10c */
	.long	0		/* 0x110 */
	.long	0		/* 0x114 */
	.long	0		/* 0x118 */
	.long	0		/* 0x11c */
	.long	0		/* 0x120 */
	.long	.kvmppc_h_bulk_remove - hcall_real_table
hcall_real_table_end:

ignore_hdec:
	mr	r4,r9
	b	fast_guest_return

_GLOBAL(kvmppc_h_set_dabr)
	std	r4,VCPU_DABR(r3)
	/* Work around P7 bug where DABR can get corrupted on mtspr */
1:	mtspr	SPRN_DABR,r4
	mfspr	r5, SPRN_DABR
	cmpd	r4, r5
	bne	1b
	isync
	li	r3,0
	blr

_GLOBAL(kvmppc_h_cede)
	ori	r11,r11,MSR_EE
	std	r11,VCPU_MSR(r3)
	li	r0,1
	stb	r0,VCPU_CEDED(r3)
	sync			/* order setting ceded vs. testing prodded */
	lbz	r5,VCPU_PRODDED(r3)
	cmpwi	r5,0
	bne	kvm_cede_prodded
	li	r0,0		/* set trap to 0 to say hcall is handled */
	stw	r0,VCPU_TRAP(r3)
	li	r0,H_SUCCESS
	std	r0,VCPU_GPR(R3)(r3)
BEGIN_FTR_SECTION
	b	kvm_cede_exit	/* just send it up to host on 970 */
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206)

	/*
	 * Set our bit in the bitmask of napping threads unless all the
	 * other threads are already napping, in which case we send this
	 * up to the host.
	 */
	ld	r5,HSTATE_KVM_VCORE(r13)
	lwz	r6,VCPU_PTID(r3)
	lwz	r8,VCORE_ENTRY_EXIT(r5)
	clrldi	r8,r8,56
	li	r0,1
	sld	r0,r0,r6
	addi	r6,r5,VCORE_NAPPING_THREADS
31:	lwarx	r4,0,r6
	or	r4,r4,r0
	PPC_POPCNTW(R7,R4)
	cmpw	r7,r8
	bge	kvm_cede_exit
	stwcx.	r4,0,r6
	bne	31b
	li	r0,1
	stb	r0,HSTATE_NAPPING(r13)
	/* order napping_threads update vs testing entry_exit_count */
	lwsync
	mr	r4,r3
	lwz	r7,VCORE_ENTRY_EXIT(r5)
	cmpwi	r7,0x100
	bge	33f		/* another thread already exiting */

/*
 * Although not specifically required by the architecture, POWER7
 * preserves the following registers in nap mode, even if an SMT mode
 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
 */
	/* Save non-volatile GPRs */
	std	r14, VCPU_GPR(R14)(r3)
	std	r15, VCPU_GPR(R15)(r3)
	std	r16, VCPU_GPR(R16)(r3)
	std	r17, VCPU_GPR(R17)(r3)
	std	r18, VCPU_GPR(R18)(r3)
	std	r19, VCPU_GPR(R19)(r3)
	std	r20, VCPU_GPR(R20)(r3)
	std	r21, VCPU_GPR(R21)(r3)
	std	r22, VCPU_GPR(R22)(r3)
	std	r23, VCPU_GPR(R23)(r3)
	std	r24, VCPU_GPR(R24)(r3)
	std	r25, VCPU_GPR(R25)(r3)
	std	r26, VCPU_GPR(R26)(r3)
	std	r27, VCPU_GPR(R27)(r3)
	std	r28, VCPU_GPR(R28)(r3)
	std	r29, VCPU_GPR(R29)(r3)
	std	r30, VCPU_GPR(R30)(r3)
	std	r31, VCPU_GPR(R31)(r3)

	/* save FP state */
	bl	.kvmppc_save_fp

	/*
	 * Take a nap until a decrementer or external interrupt occurs,
	 * with PECE1 (wake on decr) and PECE0 (wake on external) set in LPCR
	 */
	li	r0,1
	stb	r0,HSTATE_HWTHREAD_REQ(r13)
	mfspr	r5,SPRN_LPCR
	ori	r5,r5,LPCR_PECE0 | LPCR_PECE1
	mtspr	SPRN_LPCR,r5
	isync
	li	r0, 0
	std	r0, HSTATE_SCRATCH0(r13)
	ptesync
	ld	r0, HSTATE_SCRATCH0(r13)
1:	cmpd	r0, r0
	bne	1b
	nap
	b	.

kvm_end_cede:
	/* get vcpu pointer */
	ld	r4, HSTATE_KVM_VCPU(r13)

	/* Woken by external or decrementer interrupt */
	ld	r1, HSTATE_HOST_R1(r13)

	/* load up FP state */
	bl	kvmppc_load_fp

	/* Load NV GPRS */
	ld	r14, VCPU_GPR(R14)(r4)
	ld	r15, VCPU_GPR(R15)(r4)
	ld	r16, VCPU_GPR(R16)(r4)
	ld	r17, VCPU_GPR(R17)(r4)
	ld	r18, VCPU_GPR(R18)(r4)
	ld	r19, VCPU_GPR(R19)(r4)
	ld	r20, VCPU_GPR(R20)(r4)
	ld	r21, VCPU_GPR(R21)(r4)
	ld	r22, VCPU_GPR(R22)(r4)
	ld	r23, VCPU_GPR(R23)(r4)
	ld	r24, VCPU_GPR(R24)(r4)
	ld	r25, VCPU_GPR(R25)(r4)
	ld	r26, VCPU_GPR(R26)(r4)
	ld	r27, VCPU_GPR(R27)(r4)
	ld	r28, VCPU_GPR(R28)(r4)
	ld	r29, VCPU_GPR(R29)(r4)
	ld	r30, VCPU_GPR(R30)(r4)
	ld	r31, VCPU_GPR(R31)(r4)

	/* clear our bit in vcore->napping_threads */
33:	ld	r5,HSTATE_KVM_VCORE(r13)
	lwz	r3,VCPU_PTID(r4)
	li	r0,1
	sld	r0,r0,r3
	addi	r6,r5,VCORE_NAPPING_THREADS
32:	lwarx	r7,0,r6
	andc	r7,r7,r0
	stwcx.	r7,0,r6
	bne	32b
	li	r0,0
	stb	r0,HSTATE_NAPPING(r13)

	/* Check the wake reason in SRR1 to see why we got here */
	mfspr	r3, SPRN_SRR1
	rlwinm	r3, r3, 44-31, 0x7	/* extract wake reason field */
	cmpwi	r3, 4			/* was it an external interrupt? */
	li	r12, BOOK3S_INTERRUPT_EXTERNAL
	mr	r9, r4
	ld	r10, VCPU_PC(r9)
	ld	r11, VCPU_MSR(r9)
	beq	do_ext_interrupt	/* if so */

	/* see if any other thread is already exiting */
	lwz	r0,VCORE_ENTRY_EXIT(r5)
	cmpwi	r0,0x100
	blt	kvmppc_cede_reentry	/* if not go back to guest */

	/* some threads are exiting, so go to the guest exit path */
	b	hcall_real_fallback

	/* cede when already previously prodded case */
kvm_cede_prodded:
	li	r0,0
	stb	r0,VCPU_PRODDED(r3)
	sync			/* order testing prodded vs. clearing ceded */
	stb	r0,VCPU_CEDED(r3)
	li	r3,H_SUCCESS
	blr

	/* we've ceded but we want to give control to the host */
kvm_cede_exit:
	b	hcall_real_fallback

	/* Try to handle a machine check in real mode */
machine_check_realmode:
	mr	r3, r9		/* get vcpu pointer */
	bl	.kvmppc_realmode_machine_check
	nop
	cmpdi	r3, 0		/* continue exiting from guest? */
	ld	r9, HSTATE_KVM_VCPU(r13)
	li	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
	beq	mc_cont
	/* If not, deliver a machine check.  SRR0/1 are already set */
	li	r10, BOOK3S_INTERRUPT_MACHINE_CHECK
	li	r11, (MSR_ME << 1) | 1	/* synthesize MSR_SF | MSR_ME */
	rotldi	r11, r11, 63
	b	fast_interrupt_c_return

secondary_too_late:
	ld	r5,HSTATE_KVM_VCORE(r13)
	HMT_LOW
13:	lbz	r3,VCORE_IN_GUEST(r5)
	cmpwi	r3,0
	bne	13b
	HMT_MEDIUM
	ld	r11,PACA_SLBSHADOWPTR(r13)

	.rept	SLB_NUM_BOLTED
	ld	r5,SLBSHADOW_SAVEAREA(r11)
	ld	r6,SLBSHADOW_SAVEAREA+8(r11)
	andis.	r7,r5,SLB_ESID_V@h
	beq	1f
	slbmte	r6,r5
1:	addi	r11,r11,16
	.endr

secondary_nap:
	/* Clear our vcpu pointer so we don't come back in early */
	li	r0, 0
	std	r0, HSTATE_KVM_VCPU(r13)
	lwsync
	/* Clear any pending IPI - assume we're a secondary thread */
	ld	r5, HSTATE_XICS_PHYS(r13)
	li	r7, XICS_XIRR
	lwzcix	r3, r5, r7		/* ack any pending interrupt */
	rlwinm.	r0, r3, 0, 0xffffff	/* any pending? */
	beq	37f
	sync
	li	r0, 0xff
	li	r6, XICS_MFRR
	stbcix	r0, r5, r6		/* clear the IPI */
	stwcix	r3, r5, r7		/* EOI it */
37:	sync

	/* increment the nap count and then go to nap mode */
	ld	r4, HSTATE_KVM_VCORE(r13)
	addi	r4, r4, VCORE_NAP_COUNT
	lwsync				/* make previous updates visible */
51:	lwarx	r3, 0, r4
	addi	r3, r3, 1
	stwcx.	r3, 0, r4
	bne	51b

kvm_no_guest:
	li	r0, KVM_HWTHREAD_IN_NAP
	stb	r0, HSTATE_HWTHREAD_STATE(r13)

	li	r3, LPCR_PECE0
	mfspr	r4, SPRN_LPCR
	rlwimi	r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
	mtspr	SPRN_LPCR, r4
	isync
	std	r0, HSTATE_SCRATCH0(r13)
	ptesync
	ld	r0, HSTATE_SCRATCH0(r13)
1:	cmpd	r0, r0
	bne	1b
	nap
	b	.

/*
 * Save away FP, VMX and VSX registers.
 * r3 = vcpu pointer
 */
_GLOBAL(kvmppc_save_fp)
	mfmsr	r5
	ori	r8,r5,MSR_FP
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VEC@h
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
#ifdef CONFIG_VSX
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VSX@h
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	mtmsrd	r8
	isync
#ifdef CONFIG_VSX
BEGIN_FTR_SECTION
	reg = 0
	.rept	32
	li	r6,reg*16+VCPU_VSRS
	STXVD2X(reg,R6,R3)
	reg = reg + 1
	.endr
FTR_SECTION_ELSE
#endif
	reg = 0
	.rept	32
	stfd	reg,reg*8+VCPU_FPRS(r3)
	reg = reg + 1
	.endr
#ifdef CONFIG_VSX
ALT_FTR_SECTION_END_IFSET(CPU_FTR_VSX)
#endif
	mffs	fr0
	stfd	fr0,VCPU_FPSCR(r3)

#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
	reg = 0
	.rept	32
	li	r6,reg*16+VCPU_VRS
	stvx	reg,r6,r3
	reg = reg + 1
	.endr
	mfvscr	vr0
	li	r6,VCPU_VSCR
	stvx	vr0,r6,r3
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
	mfspr	r6,SPRN_VRSAVE
	stw	r6,VCPU_VRSAVE(r3)
	mtmsrd	r5
	isync
	blr

/*
 * Load up FP, VMX and VSX registers
 * r4 = vcpu pointer
 */
	.globl	kvmppc_load_fp
kvmppc_load_fp:
	mfmsr	r9
	ori	r8,r9,MSR_FP
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VEC@h
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
#ifdef CONFIG_VSX
BEGIN_FTR_SECTION
	oris	r8,r8,MSR_VSX@h
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
#endif
	mtmsrd	r8
	isync
	lfd	fr0,VCPU_FPSCR(r4)
	MTFSF_L(fr0)
#ifdef CONFIG_VSX
BEGIN_FTR_SECTION
	reg = 0
	.rept	32
	li	r7,reg*16+VCPU_VSRS
	LXVD2X(reg,R7,R4)
	reg = reg + 1
	.endr
FTR_SECTION_ELSE
#endif
	reg = 0
	.rept	32
	lfd	reg,reg*8+VCPU_FPRS(r4)
	reg = reg + 1
	.endr
#ifdef CONFIG_VSX
ALT_FTR_SECTION_END_IFSET(CPU_FTR_VSX)
#endif

#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
	li	r7,VCPU_VSCR
	lvx	vr0,r7,r4
	mtvscr	vr0
	reg = 0
	.rept	32
	li	r7,reg*16+VCPU_VRS
	lvx	reg,r7,r4
	reg = reg + 1
	.endr
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
#endif
	lwz	r7,VCPU_VRSAVE(r4)
	mtspr	SPRN_VRSAVE,r7
	blr